diff --git a/scripts/qemu-check.sh b/scripts/qemu-check.sh index cf5d3ad5d421..6c679e3c6c7f 100755 --- a/scripts/qemu-check.sh +++ b/scripts/qemu-check.sh @@ -59,7 +59,7 @@ do then READY_IPC="00 00 00 f0" SHM_IPC_REG=qemu-bridge-ipc-io - OUTBOX_OFFSET="11000" + OUTBOX_OFFSET="7000" SHM_MBOX=qemu-bridge-hp-sram-mem ROM="-r ../sof.git/build_${j}_gcc/src/arch/xtensa/rom-$j.bin" fi @@ -67,7 +67,7 @@ do then READY_IPC="00 00 00 f0" SHM_IPC_REG=qemu-bridge-ipc-io - OUTBOX_OFFSET="11000" + OUTBOX_OFFSET="7000" SHM_MBOX=qemu-bridge-hp-sram-mem ROM="-r ../sof.git/build_${j}_gcc/src/arch/xtensa/rom-$j.bin" fi @@ -75,7 +75,7 @@ do then READY_IPC="00 00 00 f0" SHM_IPC_REG=qemu-bridge-ipc-io - OUTBOX_OFFSET="11000" + OUTBOX_OFFSET="7000" SHM_MBOX=qemu-bridge-hp-sram-mem ROM="-r ../sof.git/build_${j}_gcc/src/arch/xtensa/rom-$j.bin" fi diff --git a/src/arch/host/include/arch/lib/cache.h b/src/arch/host/include/arch/lib/cache.h index a849c1c77fc8..8291675d79f3 100644 --- a/src/arch/host/include/arch/lib/cache.h +++ b/src/arch/host/include/arch/lib/cache.h @@ -12,6 +12,8 @@ #include +#define PLATFORM_DCACHE_ALIGN sizeof(void *) + static inline void dcache_writeback_region(void *addr, size_t size) {} static inline void dcache_invalidate_region(void *addr, size_t size) {} static inline void icache_invalidate_region(void *addr, size_t size) {} diff --git a/src/audio/buffer.c b/src/audio/buffer.c index dcaa3f7148a0..51525496caa3 100644 --- a/src/audio/buffer.c +++ b/src/audio/buffer.c @@ -7,54 +7,68 @@ #include #include -#include #include #include #include #include #include -#include #include #include #include #include -/* create a new component in the pipeline */ -struct comp_buffer *buffer_new(struct sof_ipc_buffer *desc) +struct comp_buffer *buffer_alloc(uint32_t size, uint32_t caps, uint32_t align) { struct comp_buffer *buffer; - trace_buffer("buffer_new()"); + trace_buffer("buffer_alloc()"); /* validate request */ - if (desc->size == 0 || desc->size > HEAP_BUFFER_SIZE) { - trace_buffer_error("buffer_new() error: " - "new size = %u is invalid", desc->size); + if (size == 0 || size > HEAP_BUFFER_SIZE) { + trace_buffer_error("buffer_alloc() error: " + "new size = %u is invalid", size); return NULL; } /* allocate new buffer */ buffer = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*buffer)); if (!buffer) { - trace_buffer_error("buffer_new() error: " + trace_buffer_error("buffer_alloc() error: " "could not alloc structure"); return NULL; } - buffer->addr = rballoc(RZONE_BUFFER, desc->caps, desc->size); + buffer->addr = rballoc_align(RZONE_BUFFER, caps, size, align); if (!buffer->addr) { rfree(buffer); - trace_buffer_error("buffer_new() error: " + trace_buffer_error("buffer_alloc() error: " "could not alloc size = %u " "bytes of type = %u", - desc->size, desc->caps); + size, caps); return NULL; } - assert(!memcpy_s(&buffer->ipc_buffer, sizeof(buffer->ipc_buffer), - desc, sizeof(*desc))); + buffer_init(buffer, size, caps); - buffer_init(buffer, desc->size); + list_init(&buffer->source_list); + list_init(&buffer->sink_list); + + return buffer; +} + +/* create a new component in the pipeline */ +struct comp_buffer *buffer_new(struct sof_ipc_buffer *desc) +{ + struct comp_buffer *buffer; + + trace_buffer("buffer_new()"); + + /* allocate buffer */ + buffer = buffer_alloc(desc->size, desc->caps, PLATFORM_DCACHE_ALIGN); + if (buffer) { + buffer->id = desc->comp.id; + buffer->pipeline_id = desc->comp.pipeline_id; + } return buffer; } @@ -62,7 +76,6 @@ struct comp_buffer *buffer_new(struct sof_ipc_buffer *desc) int buffer_set_size(struct comp_buffer *buffer, uint32_t size) { void *new_ptr = NULL; - struct sof_ipc_buffer *desc = NULL; /* validate request */ if (size == 0 || size > HEAP_BUFFER_SIZE) { @@ -70,21 +83,15 @@ int buffer_set_size(struct comp_buffer *buffer, uint32_t size) return -EINVAL; } - desc = &buffer->ipc_buffer; - if (!desc) { - trace_buffer_error("resize error: invalid buffer desc"); - return -EINVAL; - } - - if (size == desc->size) + if (size == buffer->size) return 0; - new_ptr = rbrealloc(buffer->addr, RZONE_BUFFER, desc->caps, size); + new_ptr = rbrealloc(buffer->addr, RZONE_BUFFER, buffer->caps, size); /* we couldn't allocate bigger chunk */ - if (!new_ptr && size > desc->size) { + if (!new_ptr && size > buffer->size) { trace_buffer_error("resize error: can't alloc %u bytes type %u", - desc->size, desc->caps); + buffer->size, buffer->caps); return -ENOMEM; } @@ -92,9 +99,7 @@ int buffer_set_size(struct comp_buffer *buffer, uint32_t size) if (new_ptr) buffer->addr = new_ptr; - desc->size = size; - - buffer_init(buffer, desc->size); + buffer_init(buffer, size, buffer->caps); return 0; } @@ -113,8 +118,6 @@ void buffer_free(struct comp_buffer *buffer) void comp_update_buffer_produce(struct comp_buffer *buffer, uint32_t bytes) { uint32_t flags; - uint32_t head = bytes; - uint32_t tail = 0; /* return if no bytes */ if (!bytes) { @@ -129,33 +132,6 @@ void comp_update_buffer_produce(struct comp_buffer *buffer, uint32_t bytes) irq_local_disable(flags); - /* calculate head and tail size for dcache circular wrap ops */ - if (buffer->w_ptr + bytes > buffer->end_addr) { - head = buffer->end_addr - buffer->w_ptr; - tail = bytes - head; - } - - /* - * new data produce, handle consistency for buffer and cache: - * 1. source(DMA) --> buffer --> sink(non-DMA): invalidate cache. - * 2. source(non-DMA) --> buffer --> sink(DMA): write back to memory. - * 3. source(DMA) --> buffer --> sink(DMA): do nothing. - * 4. source(non-DMA) --> buffer --> sink(non-DMA): do nothing. - */ - if (buffer->source->is_dma_connected && - !buffer->sink->is_dma_connected) { - /* need invalidate cache for sink component to use */ - dcache_invalidate_region(buffer->w_ptr, head); - if (tail) - dcache_invalidate_region(buffer->addr, tail); - } else if (!buffer->source->is_dma_connected && - buffer->sink->is_dma_connected) { - /* need write back to memory for sink component to use */ - dcache_writeback_region(buffer->w_ptr, head); - if (tail) - dcache_writeback_region(buffer->addr, tail); - } - buffer->w_ptr += bytes; /* check for pointer wrap */ @@ -184,10 +160,10 @@ void comp_update_buffer_produce(struct comp_buffer *buffer, uint32_t bytes) irq_local_enable(flags); tracev_buffer("comp_update_buffer_produce(), ((buffer->avail << 16) | " - "buffer->free) = %08x, ((buffer->ipc_buffer.comp.id << " - "16) | buffer->size) = %08x", + "buffer->free) = %08x, ((buffer->id << 16) | " + "buffer->size) = %08x", (buffer->avail << 16) | buffer->free, - (buffer->ipc_buffer.comp.id << 16) | buffer->size); + (buffer->id << 16) | buffer->size); tracev_buffer("comp_update_buffer_produce(), ((buffer->r_ptr - buffer" "->addr) << 16 | (buffer->w_ptr - buffer->addr)) = %08x", (buffer->r_ptr - buffer->addr) << 16 | @@ -229,10 +205,6 @@ void comp_update_buffer_consume(struct comp_buffer *buffer, uint32_t bytes) /* calculate free bytes */ buffer->free = buffer->size - buffer->avail; - if (buffer->sink->is_dma_connected && - !buffer->source->is_dma_connected) - dcache_writeback_region(buffer->r_ptr, bytes); - if (buffer->cb && buffer->cb_type & BUFF_CB_TYPE_CONSUME) buffer->cb(buffer->cb_data, bytes); @@ -240,11 +212,11 @@ void comp_update_buffer_consume(struct comp_buffer *buffer, uint32_t bytes) tracev_buffer("comp_update_buffer_consume(), " "(buffer->avail << 16) | buffer->free = %08x, " - "(buffer->ipc_buffer.comp.id << 16) | buffer->size = " - " %08x, (buffer->r_ptr - buffer->addr) << 16 | " + "(buffer->id << 16) | buffer->size = %08x, " + "(buffer->r_ptr - buffer->addr) << 16 | " "(buffer->w_ptr - buffer->addr)) = %08x", (buffer->avail << 16) | buffer->free, - (buffer->ipc_buffer.comp.id << 16) | buffer->size, + (buffer->id << 16) | buffer->size, (buffer->r_ptr - buffer->addr) << 16 | (buffer->w_ptr - buffer->addr)); } diff --git a/src/audio/component.c b/src/audio/component.c index 2a5513d60f71..a79925fbb512 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -98,33 +98,6 @@ void comp_unregister(struct comp_driver *drv) irq_local_enable(flags); } -int comp_set_sink_buffer(struct comp_dev *dev, uint32_t period_bytes, - uint32_t periods) -{ - struct comp_buffer *sinkb; - int ret = 0; - - sinkb = list_first_item(&dev->bsink_list, struct comp_buffer, - source_list); - - /* components connected to dma configure buffer independently */ - if (!sinkb->sink->is_dma_connected) { - ret = buffer_set_size(sinkb, period_bytes * periods); - if (ret < 0) { - trace_comp_error("comp_set_sink_buffer() error: " - "buffer_set_size() failed"); - return ret; - } - } else if (sinkb->size < period_bytes * periods) { - trace_comp_error("comp_set_sink_buffer() error: " - "sink buffer size is not sufficient"); - ret = -EINVAL; - return ret; - } - - return ret; -} - int comp_set_state(struct comp_dev *dev, int cmd) { int requested_state = comp_get_requested_state(cmd); diff --git a/src/audio/dai.c b/src/audio/dai.c index e3784b733ef7..26b491ba050d 100644 --- a/src/audio/dai.c +++ b/src/audio/dai.c @@ -65,7 +65,12 @@ struct dai_data { uint32_t frame_bytes; int xrun; /* true if we are doing xrun recovery */ + /* processing function */ + void (*process)(struct comp_buffer *source, struct comp_buffer *sink, + uint32_t bytes); + uint32_t dai_pos_blks; /* position in bytes (nearest block) */ + uint64_t start_position; /* position on start */ /* host can read back this value without IPC */ uint64_t *dai_pos; @@ -79,6 +84,7 @@ static void dai_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) struct comp_dev *dev = (struct comp_dev *)data; struct dai_data *dd = comp_get_drvdata(dev); uint32_t bytes = next->elem.size; + struct comp_buffer *local_buffer; void *buffer_ptr; tracev_dai_with_ids(dev, "dai_dma_cb()"); @@ -105,15 +111,21 @@ static void dai_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) } if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { - /* recalc available buffer space */ - comp_update_buffer_consume(dd->dma_buffer, bytes); + local_buffer = list_first_item(&dev->bsource_list, + struct comp_buffer, sink_list); + + dma_buffer_copy_to(local_buffer, dd->dma_buffer, dd->process, + bytes); - buffer_ptr = dd->dma_buffer->r_ptr; + buffer_ptr = local_buffer->r_ptr; } else { - /* recalc available buffer space */ - comp_update_buffer_produce(dd->dma_buffer, bytes); + local_buffer = list_first_item(&dev->bsink_list, + struct comp_buffer, source_list); - buffer_ptr = dd->dma_buffer->w_ptr; + dma_buffer_copy_from(dd->dma_buffer, local_buffer, dd->process, + bytes); + + buffer_ptr = local_buffer->w_ptr; } /* update host position (in bytes offset) for drivers */ @@ -209,14 +221,13 @@ static void dai_free(struct comp_dev *dev) } /* set component audio SSP and DMA configuration */ -static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) +static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes, + uint32_t period_count) { struct dai_data *dd = comp_get_drvdata(dev); struct dma_sg_config *config = &dd->config; - struct sof_ipc_comp_config *dai_config; - int err; - uint32_t buffer_size; uint32_t fifo; + int err; /* set up DMA configuration */ config->direction = DMA_DIR_MEM_TO_DEV; @@ -227,6 +238,7 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) config->dest_dev = dai_get_handshake(dd->dai, dev->params.direction, dd->stream_id); config->is_scheduling_source = comp_is_scheduling_source(dev); + config->period = dev->pipeline->ipc_pipe.period; trace_dai_with_ids(dev, "dai_playback_params() " "dest_dev = %d stream_id = %d " @@ -234,25 +246,6 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) config->dest_dev, dd->stream_id, config->src_width, config->dest_width); - /* set up local and host DMA elems to reset values */ - dai_config = COMP_GET_CONFIG(dev); - buffer_size = dai_config->periods_source * period_bytes; - - /* resize the buffer if space is available to align with period size */ - err = buffer_set_size(dd->dma_buffer, buffer_size); - if (err < 0) { - trace_dai_error_with_ids(dev, "dai_playback_params() error: " - "buffer_set_size() failed to resize " - "buffer. dai_config->periods_source =" - " %u; period_bytes = %u; " - "buffer_size = %u; " - "dd->dma_buffer->alloc_size = %u", - dai_config->periods_source, - period_bytes, buffer_size, - dd->dma_buffer->alloc_size); - return err; - } - if (!config->elem_array.elems) { fifo = dai_get_fifo(dd->dai, dev->params.direction, dd->stream_id); @@ -262,9 +255,9 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) err = dma_sg_alloc(&config->elem_array, RZONE_RUNTIME, config->direction, - dai_config->periods_source, + period_count, period_bytes, - (uintptr_t)(dd->dma_buffer->r_ptr), + (uintptr_t)(dd->dma_buffer->addr), fifo); if (err < 0) { trace_dai_error_with_ids(dev, "dai_playback_params() " @@ -277,14 +270,13 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) return 0; } -static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes) +static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes, + uint32_t period_count) { struct dai_data *dd = comp_get_drvdata(dev); struct dma_sg_config *config = &dd->config; - struct sof_ipc_comp_config *dai_config; - int err; - uint32_t buffer_size; uint32_t fifo; + int err; /* set up DMA configuration */ config->direction = DMA_DIR_DEV_TO_MEM; @@ -293,6 +285,7 @@ static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes) config->src_dev = dai_get_handshake(dd->dai, dev->params.direction, dd->stream_id); config->is_scheduling_source = comp_is_scheduling_source(dev); + config->period = dev->pipeline->ipc_pipe.period; /* TODO: Make this code platform-specific or move it driver callback */ if (dai_get_info(dd->dai, DAI_INFO_TYPE) == SOF_DAI_INTEL_DMIC) { @@ -313,25 +306,6 @@ static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes) config->src_dev, dd->stream_id, config->src_width, config->dest_width); - /* set up local and host DMA elems to reset values */ - dai_config = COMP_GET_CONFIG(dev); - buffer_size = dai_config->periods_sink * period_bytes; - - /* resize the buffer if space is available to align with period size */ - err = buffer_set_size(dd->dma_buffer, buffer_size); - if (err < 0) { - trace_dai_error_with_ids(dev, "dai_capture_params() error: " - "buffer_set_size() failed to resize " - "buffer. dai_config->periods_sink = " - "%u; period_bytes = %u; " - "buffer_size = %u; " - "dd->dma_buffer->alloc_size = %u", - dai_config->periods_sink, - period_bytes, buffer_size, - dd->dma_buffer->alloc_size); - return err; - } - if (!config->elem_array.elems) { fifo = dai_get_fifo(dd->dai, dev->params.direction, dd->stream_id); @@ -341,9 +315,9 @@ static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes) err = dma_sg_alloc(&config->elem_array, RZONE_RUNTIME, config->direction, - dai_config->periods_sink, + period_count, period_bytes, - (uintptr_t)(dd->dma_buffer->w_ptr), + (uintptr_t)(dd->dma_buffer->addr), fifo); if (err < 0) { trace_dai_error_with_ids(dev, "dai_capture_params() " @@ -360,7 +334,10 @@ static int dai_params(struct comp_dev *dev) { struct dai_data *dd = comp_get_drvdata(dev); struct sof_ipc_comp_config *dconfig = COMP_GET_CONFIG(dev); + uint32_t period_count; uint32_t period_bytes; + uint32_t buffer_size; + uint32_t addr_align; uint32_t align; int err; @@ -383,6 +360,12 @@ static int dai_params(struct comp_dev *dev) /* for DAI, we should configure its frame_fmt from topology */ dev->params.frame_fmt = dconfig->frame_fmt; + /* set processing function */ + if (dev->params.frame_fmt == SOF_IPC_FRAME_S16_LE) + dd->process = buffer_copy_s16; + else + dd->process = buffer_copy_s32; + /* calculate period size based on config */ dd->frame_bytes = comp_frame_bytes(dev); if (!dd->frame_bytes) { @@ -391,36 +374,66 @@ static int dai_params(struct comp_dev *dev) return -EINVAL; } - err = dma_get_attribute(dd->dma, DMA_ATTR_BUFFER_ALIGNMENT, &align); + err = dma_get_attribute(dd->dma, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, + &addr_align); if (err < 0) { - trace_dai_error("dai_params(): could not get dma buffer " - "alignment"); + trace_dai_error_with_ids(dev, "dai_params() error: could not " + "get dma buffer address alignment, " + "err = %d", err); return err; } - period_bytes = ALIGN_UP(dev->frames * dd->frame_bytes, align); - - if (!period_bytes) { - trace_dai_error_with_ids(dev, "dai_params() error: device has " - "no bytes (no frames to copy to sink)."); + err = dma_get_attribute(dd->dma, DMA_ATTR_BUFFER_ALIGNMENT, &align); + if (err < 0 || !align) { + trace_dai_error_with_ids(dev, "dai_params() error: could not " + "get valid dma buffer alignment, err = %d, " + "align = %u", err, align); return -EINVAL; } - if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { - dd->dma_buffer = list_first_item(&dev->bsource_list, - struct comp_buffer, - sink_list); - dd->dma_buffer->r_ptr = dd->dma_buffer->addr; + err = dma_get_attribute(dd->dma, DMA_ATTR_BUFFER_PERIOD_COUNT, + &period_count); + if (err < 0 || !period_count) { + trace_dai_error_with_ids(dev, "dai_params() error: could not " + "get valid dma buffer period count, " + "err = %d, period_count = %u", err, + period_count); + return -EINVAL; + } - return dai_playback_params(dev, period_bytes); + period_bytes = dev->frames * dd->frame_bytes; + if (!period_bytes) { + trace_dai_error_with_ids(dev, "dai_params() error: invalid " + "period_bytes."); + return -EINVAL; } - dd->dma_buffer = list_first_item(&dev->bsink_list, - struct comp_buffer, source_list); - dd->dma_buffer->w_ptr = dd->dma_buffer->addr; + /* calculate DMA buffer size */ + buffer_size = ALIGN_UP(period_count * period_bytes, align); - return dai_capture_params(dev, period_bytes); + /* alloc DMA buffer or change its size if exists */ + if (dd->dma_buffer) { + err = buffer_set_size(dd->dma_buffer, buffer_size); + if (err < 0) { + trace_dai_error_with_ids(dev, "dai_params() error: " + "buffer_set_size() failed, " + "buffer_size = %u", + buffer_size); + return err; + } + } else { + dd->dma_buffer = buffer_alloc(buffer_size, SOF_MEM_CAPS_DMA, + addr_align); + if (!dd->dma_buffer) { + trace_dai_error_with_ids(dev, "dai_params() error: " + "failed to alloc dma buffer"); + return -ENOMEM; + } + } + return dev->params.direction == SOF_IPC_STREAM_PLAYBACK ? + dai_playback_params(dev, period_bytes, period_count) : + dai_capture_params(dev, period_bytes, period_count); } static int dai_prepare(struct comp_dev *dev) @@ -473,6 +486,11 @@ static int dai_reset(struct comp_dev *dev) dma_sg_free(&config->elem_array); + if (dd->dma_buffer) { + buffer_free(dd->dma_buffer); + dd->dma_buffer = NULL; + } + dd->dai_pos_blks = 0; if (dd->dai_pos) *dd->dai_pos = 0; @@ -485,6 +503,17 @@ static int dai_reset(struct comp_dev *dev) return 0; } +static void dai_update_start_position(struct comp_dev *dev) +{ + struct dai_data *dd = comp_get_drvdata(dev); + + /* update starting wallclock */ + platform_dai_wallclock(dev, &dd->wallclock); + + /* update start position */ + dd->start_position = dev->position; +} + /* used to pass standard and bespoke command (with data) to component */ static int dai_comp_trigger(struct comp_dev *dev, int cmd) { @@ -504,11 +533,8 @@ static int dai_comp_trigger(struct comp_dev *dev, int cmd) case COMP_TRIGGER_START: trace_dai_with_ids(dev, "dai_comp_trigger(), START"); - /* only start the DAI if we are not XRUN handling - * and the pipeline is not preloaded as in this - * case start is deferred to the first copy call - */ - if (dd->xrun == 0 && !pipeline_is_preload(dev->pipeline)) { + /* only start the DAI if we are not XRUN handling */ + if (dd->xrun == 0) { /* start the DAI */ dai_trigger(dd->dai, cmd, dev->params.direction); ret = dma_start(dd->chan); @@ -518,8 +544,7 @@ static int dai_comp_trigger(struct comp_dev *dev, int cmd) dd->xrun = 0; } - /* update starting wallclock */ - platform_dai_wallclock(dev, &dd->wallclock); + dai_update_start_position(dev); break; case COMP_TRIGGER_RELEASE: /* before release, we clear the buffer data to 0s, @@ -545,8 +570,7 @@ static int dai_comp_trigger(struct comp_dev *dev, int cmd) dd->xrun = 0; } - /* update starting wallclock */ - platform_dai_wallclock(dev, &dd->wallclock); + dai_update_start_position(dev); break; case COMP_TRIGGER_XRUN: trace_dai_with_ids(dev, "dai_comp_trigger(), XRUN"); @@ -570,26 +594,27 @@ static int dai_comp_trigger(struct comp_dev *dev, int cmd) static int dai_check_for_xrun(struct comp_dev *dev, uint32_t copy_bytes) { struct dai_data *dd = comp_get_drvdata(dev); + struct comp_buffer *local_buffer; - /* data available for copy */ - if (copy_bytes) + /* data available for copy or just starting */ + if (copy_bytes || dd->start_position == dev->position) return 0; - /* no data yet, we're just starting */ - if (!dev->position) - return PPL_STATUS_PATH_STOP; - /* xrun occurred */ if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { trace_dai_error_with_ids(dev, "dai_check_for_xrun() " "error: underrun due to no data " "available"); - comp_underrun(dev, dd->dma_buffer, copy_bytes); + local_buffer = list_first_item(&dev->bsource_list, + struct comp_buffer, sink_list); + comp_underrun(dev, local_buffer, copy_bytes); } else { trace_dai_error_with_ids(dev, "dai_check_for_xrun() " "error: overrun due to no data " "available"); - comp_overrun(dev, dd->dma_buffer, copy_bytes); + local_buffer = list_first_item(&dev->bsink_list, + struct comp_buffer, source_list); + comp_overrun(dev, local_buffer, copy_bytes); } return -ENODATA; @@ -599,6 +624,7 @@ static int dai_check_for_xrun(struct comp_dev *dev, uint32_t copy_bytes) static int dai_copy(struct comp_dev *dev) { struct dai_data *dd = comp_get_drvdata(dev); + struct comp_buffer *local_buffer; uint32_t avail_bytes = 0; uint32_t free_bytes = 0; uint32_t copy_bytes = 0; @@ -606,34 +632,27 @@ static int dai_copy(struct comp_dev *dev) tracev_dai_with_ids(dev, "dai_copy()"); - /* start DMA on preload */ - if (pipeline_is_preload(dev->pipeline)) { - /* start the DAI */ - dai_trigger(dd->dai, COMP_TRIGGER_START, - dev->params.direction); - ret = dma_start(dd->chan); - if (ret < 0) - return ret; - platform_dai_wallclock(dev, &dd->wallclock); - - return 0; - } - /* get data sizes from DMA */ ret = dma_get_data_size(dd->chan, &avail_bytes, &free_bytes); if (ret < 0) return ret; /* calculate minimum size to copy */ - copy_bytes = dev->params.direction == SOF_IPC_STREAM_PLAYBACK ? - MIN(dd->dma_buffer->avail, free_bytes) : - MIN(avail_bytes, dd->dma_buffer->free); + if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { + local_buffer = list_first_item(&dev->bsource_list, + struct comp_buffer, sink_list); + copy_bytes = MIN(local_buffer->avail, free_bytes); + } else { + local_buffer = list_first_item(&dev->bsink_list, + struct comp_buffer, source_list); + copy_bytes = MIN(avail_bytes, local_buffer->free); + } tracev_dai_with_ids(dev, "dai_copy(), copy_bytes = 0x%x", copy_bytes); /* check for underrun or overrun */ ret = dai_check_for_xrun(dev, copy_bytes); - if (ret < 0 || ret == PPL_STATUS_PATH_STOP) + if (ret < 0) return ret; ret = dma_copy(dd->chan, copy_bytes, 0); @@ -808,7 +827,6 @@ static int dai_config(struct comp_dev *dev, struct sof_ipc_dai_config *config) /* set up callback */ dma_set_cb(dd->chan, DMA_CB_TYPE_COPY, dai_dma_cb, dev); - dev->is_dma_connected = 1; } return 0; diff --git a/src/audio/eq_fir/eq_fir.c b/src/audio/eq_fir/eq_fir.c index 4b1a27a9c367..f246618307c8 100644 --- a/src/audio/eq_fir/eq_fir.c +++ b/src/audio/eq_fir/eq_fir.c @@ -733,12 +733,10 @@ static int eq_fir_prepare(struct comp_dev *dev) else dev->params.frame_fmt = cd->sink_format; - /* set downstream buffer size */ - ret = comp_set_sink_buffer(dev, sink_period_bytes, - config->periods_sink); - if (ret < 0) { + if (sinkb->size < config->periods_sink * sink_period_bytes) { trace_eq_error("eq_fir_prepare() error: " - "comp_set_sink_buffer() failed"); + "sink buffer size is insufficient"); + ret = -ENOMEM; goto err; } diff --git a/src/audio/eq_iir/eq_iir.c b/src/audio/eq_iir/eq_iir.c index beeeb14a53d9..188b8d04daa3 100644 --- a/src/audio/eq_iir/eq_iir.c +++ b/src/audio/eq_iir/eq_iir.c @@ -778,12 +778,10 @@ static int eq_iir_prepare(struct comp_dev *dev) else dev->params.frame_fmt = cd->sink_format; - /* set downstream buffer size */ - ret = comp_set_sink_buffer(dev, sink_period_bytes, - config->periods_sink); - if (ret < 0) { + if (sinkb->size < config->periods_sink * sink_period_bytes) { trace_eq_error("eq_iir_prepare() error: " - "comp_set_sink_buffer() failed"); + "sink buffer size is insufficient"); + ret = -ENOMEM; goto err; } diff --git a/src/audio/host.c b/src/audio/host.c index 0e2e46c0a3a4..23dcb066a950 100644 --- a/src/audio/host.c +++ b/src/audio/host.c @@ -77,7 +77,6 @@ struct host_data { struct dma_chan_data *chan; struct dma_sg_config config; struct comp_buffer *dma_buffer; - uint32_t period_bytes; /**< Size of a single period (in bytes) */ /* host position reporting related */ uint32_t host_size; /**< Host buffer size (in bytes) */ @@ -95,13 +94,14 @@ struct host_data { struct hc_buf *source; struct hc_buf *sink; - /* helper used in split transactions */ - uint32_t split_value; - uint32_t dma_copy_align; /**< Minimal chunk of data possible to be * copied by dma connected to host */ + /* processing function */ + void (*process)(struct comp_buffer *source, struct comp_buffer *sink, + uint32_t bytes); + /* stream info */ struct sof_ipc_stream_posn posn; /* TODO: update this */ }; @@ -136,11 +136,19 @@ static uint32_t host_dma_get_split(struct host_data *hd, uint32_t bytes) static void host_update_position(struct comp_dev *dev, uint32_t bytes) { struct host_data *hd = comp_get_drvdata(dev); + struct comp_buffer *local_buffer; - if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) - comp_update_buffer_produce(hd->dma_buffer, bytes); - else - comp_update_buffer_consume(hd->dma_buffer, bytes); + if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { + local_buffer = list_first_item(&dev->bsink_list, + struct comp_buffer, source_list); + dma_buffer_copy_from(hd->dma_buffer, local_buffer, hd->process, + bytes); + } else { + local_buffer = list_first_item(&dev->bsource_list, + struct comp_buffer, sink_list); + dma_buffer_copy_to(local_buffer, hd->dma_buffer, hd->process, + bytes); + } dev->position += bytes; @@ -291,10 +299,10 @@ static int host_trigger(struct comp_dev *dev, int cmd) return ret; } - /* we should ignore any trigger commands when doing one shot, - * because transfers will start in copy and stop automatically + /* we should ignore any trigger commands besides start + * when doing one shot, because transfers will stop automatically */ - if (hd->copy_type == COMP_COPY_ONE_SHOT) + if (cmd != COMP_TRIGGER_START && hd->copy_type == COMP_COPY_ONE_SHOT) return ret; if (!hd->chan) { @@ -378,7 +386,6 @@ static struct comp_dev *host_new(struct sof_ipc_comp *comp) hd->copy_type = COMP_COPY_NORMAL; hd->posn.comp_id = comp->id; dev->state = COMP_STATE_READY; - dev->is_dma_connected = 1; return dev; } @@ -430,104 +437,15 @@ static int host_elements_reset(struct comp_dev *dev) return 0; } -static uint32_t host_buffer_get_copy_bytes(struct comp_dev *dev) -{ - struct host_data *hd = comp_get_drvdata(dev); - struct dma_sg_elem *local_elem = hd->config.elem_array.elems; - uint32_t avail_bytes = 0; - uint32_t free_bytes = 0; - uint32_t copy_bytes = 0; - int ret; - - if (hd->copy_type == COMP_COPY_ONE_SHOT) { - /* calculate minimum size to copy */ - copy_bytes = dev->params.direction == SOF_IPC_STREAM_PLAYBACK ? - hd->dma_buffer->free : hd->dma_buffer->avail; - - /* copy_bytes should be aligned to minimum possible chunk of - * data to be copied by dma. - */ - copy_bytes = ALIGN_DOWN(copy_bytes, hd->dma_copy_align); - - hd->split_value = host_dma_get_split(hd, copy_bytes); - if (hd->split_value) - copy_bytes -= hd->split_value; - - local_elem->size = copy_bytes; - } else { - /* get data sizes from DMA */ - ret = dma_get_data_size(hd->chan, &avail_bytes, - &free_bytes); - if (ret < 0) { - trace_host_error("host_buffer_cb() error: " - "dma_get_data_size() failed, ret = %u", - ret); - return 0; - } - - /* calculate minimum size to copy */ - copy_bytes = dev->params.direction == SOF_IPC_STREAM_PLAYBACK ? - MIN(avail_bytes, hd->dma_buffer->free) : - MIN(hd->dma_buffer->avail, free_bytes); - - /* copy_bytes should be aligned to minimum possible chunk of - * data to be copied by dma. - */ - copy_bytes = ALIGN_DOWN(copy_bytes, hd->dma_copy_align); - } - - return copy_bytes; -} - -static void host_buffer_cb(void *data, uint32_t bytes) -{ - struct comp_dev *dev = (struct comp_dev *)data; - struct host_data *hd = comp_get_drvdata(dev); - uint32_t copy_bytes = 0; - uint32_t flags = 0; - int ret; - - tracev_host("host_buffer_cb(), copy_bytes = 0x%x", copy_bytes); - - if (hd->copy_type == COMP_COPY_BLOCKING) - flags |= DMA_COPY_BLOCKING; - else if (hd->copy_type == COMP_COPY_ONE_SHOT) - flags |= DMA_COPY_ONE_SHOT; - - /* copy including all split transfers */ - do { - copy_bytes = host_buffer_get_copy_bytes(dev); - - /* reconfigure transfer */ - ret = dma_set_config(hd->chan, &hd->config); - if (ret < 0) { - trace_host_error("host_buffer_cb() error: " - "dma_set_config() failed, ret = %u", - ret); - hd->split_value = 0; - return; - } - - ret = dma_copy(hd->chan, copy_bytes, flags); - if (ret < 0) { - trace_host_error("host_buffer_cb() error: dma_copy() " - "failed, ret = %u", ret); - hd->split_value = 0; - return; - } - } while (hd->split_value); -} - /* configure the DMA params and descriptors for host buffer IO */ static int host_params(struct comp_dev *dev) { struct host_data *hd = comp_get_drvdata(dev); - struct sof_ipc_comp_config *cconfig = COMP_GET_CONFIG(dev); struct dma_sg_config *config = &hd->config; - uint32_t buffer_size; - uint32_t buffer_count; - uint32_t buffer_single_size; uint32_t period_count; + uint32_t period_bytes; + uint32_t buffer_size; + uint32_t addr_align; uint32_t align; int err; @@ -536,87 +454,85 @@ static int host_params(struct comp_dev *dev) /* host params always installed by pipeline IPC */ hd->host_size = dev->params.buffer.size; - err = dma_get_attribute(hd->dma, DMA_ATTR_BUFFER_ALIGNMENT, &align); + /* retrieve DMA buffer address alignment */ + err = dma_get_attribute(hd->dma, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, + &addr_align); if (err < 0) { - trace_host_error_with_ids(dev, "could not get dma buffer " - "alignment"); + trace_host_error_with_ids(dev, "host_params() error: could not " + "get dma buffer address alignment, " + "err = %d", err); return err; } - /* determine source and sink buffer elems */ - if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { - hd->dma_buffer = list_first_item(&dev->bsink_list, - struct comp_buffer, source_list); - - /* set callback on buffer consume */ - buffer_set_cb(hd->dma_buffer, &host_buffer_cb, dev, - BUFF_CB_TYPE_CONSUME); - - config->direction = DMA_DIR_HMEM_TO_LMEM; - - period_count = cconfig->periods_sink; + /* retrieve DMA buffer size alignment */ + err = dma_get_attribute(hd->dma, DMA_ATTR_BUFFER_ALIGNMENT, &align); + if (err < 0 || !align) { + trace_host_error_with_ids(dev, "host_params() error: could not " + "get valid dma buffer alignment, err " + "= %d, align = %u", err, align); + return -EINVAL; + } - buffer_count = hd->host.elem_array.count ? 1 : period_count; - buffer_single_size = ALIGN_UP(dev->frames * - comp_frame_bytes(dev), - align); + /* retrieve DMA buffer period count */ + err = dma_get_attribute(hd->dma, DMA_ATTR_BUFFER_PERIOD_COUNT, + &period_count); + if (err < 0 || !period_count) { + trace_host_error_with_ids(dev, "host_params() error: could not " + "get valid dma buffer period count, " + "err = %d, period_count = %u", err, + period_count); + return -EINVAL; + } - if (hd->host.elem_array.count) - buffer_single_size *= period_count; + period_bytes = dev->frames * comp_frame_bytes(dev); + if (!period_bytes) { + trace_host_error_with_ids(dev, "host_params() error: invalid " + "period_bytes"); + return -EINVAL; + } + /* determine source and sink buffer elements */ + if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { + config->direction = DMA_DIR_HMEM_TO_LMEM; hd->source = &hd->host; hd->sink = &hd->local; } else { - hd->dma_buffer = list_first_item(&dev->bsource_list, - struct comp_buffer, sink_list); - - /* set callback on buffer produce */ - buffer_set_cb(hd->dma_buffer, &host_buffer_cb, dev, - BUFF_CB_TYPE_PRODUCE); - config->direction = DMA_DIR_LMEM_TO_HMEM; - - period_count = cconfig->periods_source; - - buffer_count = hd->host.elem_array.count ? 1 : period_count; - buffer_single_size = ALIGN_UP(dev->frames * - comp_frame_bytes(dev), - align); - - if (hd->host.elem_array.count) - buffer_single_size *= period_count; - hd->source = &hd->local; hd->sink = &hd->host; } - /* validate period count */ - if (!period_count) { - trace_host_error_with_ids(dev, "host_params() error: invalid " - "period_count"); - return -EINVAL; + /* TODO: should be taken from DMA */ + if (hd->host.elem_array.count) { + period_bytes *= period_count; + period_count = 1; } - hd->period_bytes = ALIGN_UP(dev->frames * comp_frame_bytes(dev), align); + /* calculate DMA buffer size */ + buffer_size = ALIGN_UP(period_count * period_bytes, align); - if (hd->period_bytes == 0) { - trace_host_error_with_ids(dev, "host_params() error: invalid " - "period_bytes"); - return -EINVAL; - } - - /* resize the buffer if space is available to align with period size */ - buffer_size = period_count * hd->period_bytes; - err = buffer_set_size(hd->dma_buffer, buffer_size); - if (err < 0) { - trace_host_error_with_ids(dev, "host_params() error:" - "buffer_set_size() failed, " - "buffer_size = %u", buffer_size); - return err; + /* alloc DMA buffer or change its size if exists */ + if (hd->dma_buffer) { + err = buffer_set_size(hd->dma_buffer, buffer_size); + if (err < 0) { + trace_host_error_with_ids(dev, "host_params() error: " + "buffer_set_size() failed, " + "buffer_size = %u", + buffer_size); + return err; + } + } else { + hd->dma_buffer = buffer_alloc(buffer_size, SOF_MEM_CAPS_DMA, + addr_align); + if (!hd->dma_buffer) { + trace_host_error_with_ids(dev, "host_params() error: " + "failed to alloc dma buffer"); + return -ENOMEM; + } } /* create SG DMA elems for local DMA buffer */ - err = create_local_elems(dev, buffer_count, buffer_single_size); + err = create_local_elems(dev, period_count, buffer_size / period_count); if (err < 0) return err; @@ -626,6 +542,7 @@ static int host_params(struct comp_dev *dev) config->cyclic = 0; config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); config->is_scheduling_source = comp_is_scheduling_source(dev); + config->period = dev->pipeline->ipc_pipe.period; host_elements_reset(dev); @@ -662,6 +579,12 @@ static int host_params(struct comp_dev *dev) /* set up callback */ dma_set_cb(hd->chan, DMA_CB_TYPE_COPY, host_dma_cb, dev); + /* set processing function */ + if (dev->params.frame_fmt == SOF_IPC_FRAME_S16_LE) + hd->process = buffer_copy_s16; + else + hd->process = buffer_copy_s32; + return 0; } @@ -681,7 +604,6 @@ static int host_prepare(struct comp_dev *dev) hd->local_pos = 0; hd->report_pos = 0; - hd->split_value = 0; dev->position = 0; return 0; @@ -724,6 +646,12 @@ static int host_reset(struct comp_dev *dev) dma_sg_free(&hd->local.elem_array); dma_sg_free(&hd->config.elem_array); + /* free DMA buffer */ + if (hd->dma_buffer) { + buffer_free(hd->dma_buffer); + hd->dma_buffer = NULL; + } + /* reset dma channel as we have put it */ hd->chan = NULL; @@ -736,10 +664,80 @@ static int host_reset(struct comp_dev *dev) return 0; } +static uint32_t host_buffer_get_copy_bytes(struct comp_dev *dev) +{ + struct host_data *hd = comp_get_drvdata(dev); + struct dma_sg_elem *local_elem = hd->config.elem_array.elems; + struct comp_buffer *local_buffer; + uint32_t avail_bytes = 0; + uint32_t free_bytes = 0; + uint32_t copy_bytes = 0; + uint32_t split_value; + int ret; + + if (hd->copy_type == COMP_COPY_ONE_SHOT) { + /* calculate minimum size to copy */ + if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { + local_buffer = list_first_item(&dev->bsink_list, + struct comp_buffer, + source_list); + copy_bytes = local_buffer->free; + } else { + local_buffer = list_first_item(&dev->bsource_list, + struct comp_buffer, + sink_list); + copy_bytes = local_buffer->avail; + } + + /* copy_bytes should be aligned to minimum possible chunk of + * data to be copied by dma. + */ + copy_bytes = ALIGN_DOWN(copy_bytes, hd->dma_copy_align); + + split_value = host_dma_get_split(hd, copy_bytes); + if (split_value) + copy_bytes -= split_value; + + local_elem->size = copy_bytes; + } else { + /* get data sizes from DMA */ + ret = dma_get_data_size(hd->chan, &avail_bytes, + &free_bytes); + if (ret < 0) { + trace_host_error("host_buffer_cb() error: " + "dma_get_data_size() failed, ret = %u", + ret); + return 0; + } + + /* calculate minimum size to copy */ + if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK) { + local_buffer = list_first_item(&dev->bsink_list, + struct comp_buffer, + source_list); + copy_bytes = MIN(avail_bytes, local_buffer->free); + } else { + local_buffer = list_first_item(&dev->bsource_list, + struct comp_buffer, + sink_list); + copy_bytes = MIN(local_buffer->avail, free_bytes); + } + + /* copy_bytes should be aligned to minimum possible chunk of + * data to be copied by dma. + */ + copy_bytes = ALIGN_DOWN(copy_bytes, hd->dma_copy_align); + } + + return copy_bytes; +} + /* copy and process stream data from source to sink buffers */ static int host_copy(struct comp_dev *dev) { struct host_data *hd = comp_get_drvdata(dev); + uint32_t copy_bytes = 0; + uint32_t flags = 0; int ret = 0; tracev_host_with_ids(dev, "host_copy()"); @@ -747,26 +745,35 @@ static int host_copy(struct comp_dev *dev) if (dev->state != COMP_STATE_ACTIVE) return 0; - /* here only do preload, further copies happen - * in host_buffer_cb() - */ - if (dev->params.direction == SOF_IPC_STREAM_PLAYBACK && - !dev->position) { - ret = dma_copy(hd->chan, hd->dma_buffer->size, - DMA_COPY_PRELOAD); - if (ret < 0) { - if (ret == -ENODATA) { - /* preload not finished, so stop processing */ - trace_host_with_ids(dev, "host_copy(), preload" - " not yet finished"); - return PPL_STATUS_PATH_STOP; - } - - trace_host_error_with_ids(dev, "host_copy() error: " - "dma_copy() failed, " - "ret = %u", ret); - return ret; - } + if (hd->copy_type == COMP_COPY_BLOCKING) + flags |= DMA_COPY_BLOCKING; + else if (hd->copy_type == COMP_COPY_ONE_SHOT) + flags |= DMA_COPY_ONE_SHOT; + + /* update first transfer manually */ + if (!dev->position && flags & COMP_COPY_ONE_SHOT) + host_one_shot_cb(dev, hd->dma_buffer->size); + + copy_bytes = host_buffer_get_copy_bytes(dev); + if (!copy_bytes) { + trace_host_with_ids(dev, "host_copy(): no bytes to copy"); + return ret; + } + + /* reconfigure transfer */ + ret = dma_set_config(hd->chan, &hd->config); + if (ret < 0) { + trace_host_error("host_copy() error: " + "dma_set_config() failed, ret = %u", + ret); + return ret; + } + + ret = dma_copy(hd->chan, copy_bytes, flags); + if (ret < 0) { + trace_host_error("host_copy() error: dma_copy() " + "failed, ret = %u", ret); + return ret; } return ret; diff --git a/src/audio/kpb.c b/src/audio/kpb.c index 23b7115b596b..8ca6c95b4927 100644 --- a/src/audio/kpb.c +++ b/src/audio/kpb.c @@ -625,7 +625,6 @@ static int kpb_copy(struct comp_dev *dev) comp_update_buffer_produce(sink, copy_bytes); comp_update_buffer_consume(source, copy_bytes); - ret = PPL_STATUS_PATH_STOP; break; case KPB_STATE_DRAINING: /* In draining state we only buffer data in internal, @@ -1098,8 +1097,10 @@ static enum task_state kpb_draining_task(void *arg) move_buffer = false; } - if (size_to_copy) + if (size_to_copy) { comp_update_buffer_produce(sink, size_to_copy); + comp_copy(sink->sink); + } if (sync_mode_on && period_bytes >= period_bytes_limit) { current_time = platform_timer_get(platform_timer); diff --git a/src/audio/mixer.c b/src/audio/mixer.c index ce2db2303286..655c19dd47a6 100644 --- a/src/audio/mixer.c +++ b/src/audio/mixer.c @@ -154,8 +154,8 @@ static void mixer_free(struct comp_dev *dev) static int mixer_params(struct comp_dev *dev) { struct sof_ipc_comp_config *config = COMP_GET_CONFIG(dev); + struct comp_buffer *sinkb; uint32_t period_bytes; - int ret; trace_mixer("mixer_params()"); @@ -166,13 +166,13 @@ static int mixer_params(struct comp_dev *dev) return -EINVAL; } - /* set downstream buffer size */ - ret = comp_set_sink_buffer(dev, period_bytes, - config->periods_sink); - if (ret < 0) { + sinkb = list_first_item(&dev->bsink_list, struct comp_buffer, + source_list); + + if (sinkb->size < config->periods_sink * period_bytes) { trace_mixer_error("mixer_params() error: " - "comp_set_sink_buffer() failed"); - return ret; + "sink buffer size is insufficient"); + return -ENOMEM; } return 0; diff --git a/src/audio/mux/mux.c b/src/audio/mux/mux.c index 8da0b0a45659..e9d2c84e3068 100644 --- a/src/audio/mux/mux.c +++ b/src/audio/mux/mux.c @@ -228,7 +228,7 @@ static int demux_copy(struct comp_dev *dev) sink = container_of(clist, struct comp_buffer, source_list); if (sink->sink->state == dev->state) { num_sinks++; - i = get_stream_index(cd, sink->ipc_buffer.comp.pipeline_id); + i = get_stream_index(cd, sink->pipeline_id); sinks[i] = sink; } } @@ -297,7 +297,7 @@ static int mux_copy(struct comp_dev *dev) source = container_of(clist, struct comp_buffer, sink_list); if (source->source->state == dev->state) { num_sources++; - i = get_stream_index(cd, source->ipc_buffer.comp.pipeline_id); + i = get_stream_index(cd, source->pipeline_id); sources[i] = source; } } diff --git a/src/audio/pipeline.c b/src/audio/pipeline.c index 7c8e0566116f..34d25e7376d3 100644 --- a/src/audio/pipeline.c +++ b/src/audio/pipeline.c @@ -38,8 +38,6 @@ struct pipeline_data { }; static enum task_state pipeline_task(void *arg); -static enum task_state pipeline_preload_task(void *arg); -static void pipeline_schedule_preload(struct pipeline *p); /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, @@ -72,7 +70,7 @@ int pipeline_connect(struct comp_dev *comp, struct comp_buffer *buffer, uint32_t flags; trace_pipe("pipeline: connect comp %d and buffer %d", - comp->comp.id, buffer->ipc_buffer.comp.id); + comp->comp.id, buffer->id); irq_local_disable(flags); list_item_prepend(buffer_comp_list(buffer, dir), @@ -234,9 +232,6 @@ int pipeline_free(struct pipeline *p) if (p->pipe_task) schedule_task_free(p->pipe_task); - if (p->preload_task) - schedule_task_free(p->preload_task); - /* now free the pipeline */ rfree(p); @@ -350,6 +345,48 @@ int pipeline_params(struct pipeline *p, struct comp_dev *host, return ret; } +static struct task *pipeline_task_init(struct pipeline *p, uint32_t type, + enum task_state (*func)(void *data)) +{ + struct task *task = NULL; + + task = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*task)); + + if (!task) + return NULL; + + if (schedule_task_init(task, type, p->ipc_pipe.priority, func, + NULL, p, p->ipc_pipe.core, 0) < 0) { + rfree(task); + task = NULL; + } + + return task; +} + +static int pipeline_comp_task_init(struct pipeline *p) +{ + uint32_t type; + + /* initialize task if necessary */ + if (!p->pipe_task) { + /* right now we always consider pipeline as a low latency + * component, but it may change in the future + */ + type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL_TIMER : + SOF_SCHEDULE_LL_DMA; + + p->pipe_task = pipeline_task_init(p, type, pipeline_task); + if (!p->pipe_task) { + trace_pipe_error("pipeline_prepare() error: task init " + "failed"); + return -ENOMEM; + } + } + + return 0; +} + static int pipeline_comp_prepare(struct comp_dev *current, void *data, int dir) { int err = 0; @@ -381,6 +418,10 @@ static int pipeline_comp_prepare(struct comp_dev *current, void *data, int dir) } } + err = pipeline_comp_task_init(current->pipeline); + if (err < 0) + return err; + err = comp_prepare(current); if (err < 0 || err == PPL_STATUS_PATH_STOP) return err; @@ -389,25 +430,10 @@ static int pipeline_comp_prepare(struct comp_dev *current, void *data, int dir) &buffer_reset_pos, dir); } -static struct task *pipeline_task_init(struct pipeline *p, uint32_t type, - enum task_state (*func)(void *data)) -{ - struct task *task = NULL; - - task = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*task)); - - if (task) - schedule_task_init(task, type, p->ipc_pipe.priority, func, NULL, - p, p->ipc_pipe.core, 0); - - return task; -} - -/* prepare the pipeline for usage - preload host buffers here */ +/* prepare the pipeline for usage */ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) { struct pipeline_data ppl_data; - uint32_t type; int ret = 0; trace_pipe_with_ids(p, "pipeline_prepare()"); @@ -421,40 +447,6 @@ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) return ret; } - /* pipeline preload needed only for playback streams without active - * sink component (it can be active for e.g. mixer pipelines) - */ - p->preload = dev->params.direction == SOF_IPC_STREAM_PLAYBACK && - p->sink_comp->state != COMP_STATE_ACTIVE; - - /* initialize task if necessary */ - if (!p->pipe_task) { - /* right now we always consider pipeline as a low latency - * component, but it may change in the future - */ - type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL_TIMER : - SOF_SCHEDULE_LL_DMA; - - p->pipe_task = pipeline_task_init(p, type, pipeline_task); - if (!p->pipe_task) { - trace_pipe_error("pipeline_prepare() error: task init " - "failed"); - return -ENOMEM; - } - } - - /* initialize preload task if necessary */ - if (p->preload && !p->preload_task) { - /* preload task is always EDF as it guarantees scheduling */ - p->preload_task = pipeline_task_init(p, SOF_SCHEDULE_EDF, - pipeline_preload_task); - if (!p->preload_task) { - trace_pipe_error("pipeline_prepare() error: preload " - "task init failed"); - return -ENOMEM; - } - } - p->status = COMP_STATE_PREPARE; return ret; @@ -490,8 +482,6 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) if (cmd == CACHE_INVALIDATE) { dcache_invalidate_region(p, sizeof(*p)); dcache_invalidate_region(p->pipe_task, sizeof(*p->pipe_task)); - dcache_invalidate_region(p->preload_task, - sizeof(*p->preload_task)); } trace_pipe_with_ids(p, "pipeline_cache()"); @@ -508,8 +498,6 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) if (cmd == CACHE_WRITEBACK_INV) { dcache_writeback_invalidate_region(p->pipe_task, sizeof(*p->pipe_task)); - dcache_writeback_invalidate_region(p->preload_task, - sizeof(*p->preload_task)); dcache_writeback_invalidate_region(p, sizeof(*p)); } @@ -532,15 +520,8 @@ static void pipeline_comp_trigger_sched_comp(struct pipeline *p, break; case COMP_TRIGGER_RELEASE: case COMP_TRIGGER_START: + pipeline_schedule_copy(p, 0); p->xrun_bytes = 0; - - /* schedule preload if pipeline is not timer driven */ - if (p->preload && !pipeline_is_timer_driven(p)) - /* schedule pipeline preload */ - pipeline_schedule_preload(p); - else - pipeline_schedule_copy(p, 0); - p->status = COMP_STATE_ACTIVE; break; case COMP_TRIGGER_SUSPEND: @@ -669,7 +650,6 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) { struct pipeline_data data; int ret; - uint32_t flags; trace_pipe_with_ids(p, "pipeline_trigger()"); @@ -692,8 +672,6 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) data.start = host; data.cmd = cmd; - irq_local_disable(flags); - ret = pipeline_comp_trigger(host, &data, host->params.direction); if (ret < 0) { trace_ipc_error("pipeline_trigger() error: ret = %d, host->" @@ -701,7 +679,6 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) cmd); } - irq_local_enable(flags); return ret; } @@ -802,10 +779,8 @@ static int pipeline_comp_copy(struct comp_dev *current, void *data, int dir) /* Copy data across all pipeline components. * For capture pipelines it always starts from source component - * and continues downstream. For playback pipelines there are two - * possibilities: for preload it starts from sink component and - * continues upstream and if not preload, then it first copies - * sink component itself and then goes upstream. + * and continues downstream and for playback pipelines it first + * copies sink component itself and then goes upstream. */ static int pipeline_copy(struct pipeline *p) { @@ -817,21 +792,6 @@ static int pipeline_copy(struct pipeline *p) if (p->source_comp->params.direction == SOF_IPC_STREAM_PLAYBACK) { dir = PPL_DIR_UPSTREAM; start = p->sink_comp; - - /* if not pipeline preload then copy sink comp first */ - if (!p->preload) { - ret = comp_copy(start); - if (ret < 0) { - trace_pipe_error("pipeline_copy() error: " - "ret = %d", ret); - return ret; - } - - start = comp_get_previous(start, dir); - if (!start) - /* nothing else to do */ - return ret; - } } else { dir = PPL_DIR_DOWNSTREAM; start = p->source_comp; @@ -846,12 +806,6 @@ static int pipeline_copy(struct pipeline *p) "->comp.id = %u, dir = %u", ret, start->comp.id, dir); - /* stop preload only after full walkthrough */ - if (ret != PPL_STATUS_PATH_STOP && p->preload) { - p->preload = false; - pipeline_schedule_copy(p, 0); - } - return ret; } @@ -999,11 +953,6 @@ void pipeline_schedule_cancel(struct pipeline *p) schedule_task_cancel(p->pipe_task); } -static void pipeline_schedule_preload(struct pipeline *p) -{ - schedule_task(p->preload_task, 0, p->ipc_pipe.period); -} - static enum task_state pipeline_task(void *arg) { struct pipeline *p = arg; @@ -1037,17 +986,3 @@ static enum task_state pipeline_task(void *arg) return SOF_TASK_STATE_RESCHEDULE; } - -static enum task_state pipeline_preload_task(void *arg) -{ - struct pipeline *p = arg; - - if (pipeline_copy(p) < 0) { - trace_pipe_error_with_ids(p, "pipeline_preload_task(): preload " - "has failed"); - return SOF_TASK_STATE_COMPLETED; - } - - return p->preload ? SOF_TASK_STATE_RESCHEDULE : - SOF_TASK_STATE_COMPLETED; -} diff --git a/src/audio/selector/selector.c b/src/audio/selector/selector.c index 79bd35955b68..249c86f98a5a 100644 --- a/src/audio/selector/selector.c +++ b/src/audio/selector/selector.c @@ -380,12 +380,10 @@ static int selector_prepare(struct comp_dev *dev) trace_selector("selector_prepare(): sink->params.channels = %u", sinkb->sink->params.channels); - /* set downstream buffer size */ - ret = comp_set_sink_buffer(dev, cd->sink_period_bytes, - config->periods_sink); - if (ret < 0) { + if (sinkb->size < config->periods_sink * cd->sink_period_bytes) { trace_selector_error("selector_prepare() error: " - "comp_set_sink_buffer() failed"); + "sink buffer size is insufficient"); + ret = -ENOMEM; goto err; } diff --git a/src/audio/src/src.c b/src/audio/src/src.c index 11890dcbe59f..7a19c3f3371e 100644 --- a/src/audio/src/src.c +++ b/src/audio/src/src.c @@ -850,12 +850,10 @@ static int src_prepare(struct comp_dev *dev) else dev->params.frame_fmt = cd->sink_format; - /* set downstream buffer size */ - ret = comp_set_sink_buffer(dev, sink_period_bytes, - config->periods_sink); - if (ret < 0) { + if (sinkb->size < config->periods_sink * sink_period_bytes) { trace_src_error("src_prepare() error: " - "comp_set_sink_buffer() failed"); + "sink buffer size is insufficient"); + ret = -ENOMEM; goto err; } diff --git a/src/audio/volume/volume.c b/src/audio/volume/volume.c index 5ed4368a0eb3..3f5180b8a26d 100644 --- a/src/audio/volume/volume.c +++ b/src/audio/volume/volume.c @@ -615,14 +615,10 @@ static int volume_prepare(struct comp_dev *dev) else dev->params.frame_fmt = cd->sink_format; - /* set downstream buffer size */ - ret = comp_set_sink_buffer(dev, sink_period_bytes, - config->periods_sink); - - if (ret < 0) { + if (sinkb->size < config->periods_sink * sink_period_bytes) { trace_volume_error("volume_prepare() error: " - "comp_set_sink_buffer() failed"); - + "sink buffer size is insufficient"); + ret = -ENOMEM; goto err; } diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 2d085ac1fe9a..c7916838af67 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -65,6 +65,12 @@ struct dw_dma_chan_data { */ static const uint32_t burst_elems[] = {1, 2, 4, 8}; +#if CONFIG_HW_LLI +#define DW_DMA_BUFFER_PERIOD_COUNT 3 +#else +#define DW_DMA_BUFFER_PERIOD_COUNT 2 +#endif + static void dw_dma_interrupt_mask(struct dma_chan_data *channel) { /* mask block, transfer and error interrupts for channel */ @@ -462,6 +468,7 @@ static int dw_dma_set_config(struct dma_chan_data *channel, /* default channel config */ channel->direction = config->direction; channel->is_scheduling_source = config->is_scheduling_source; + channel->period = config->period; dw_chan->cfg_lo = DW_CFG_LOW_DEF; dw_chan->cfg_hi = DW_CFG_HIGH_DEF; @@ -840,12 +847,21 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, .status = DMA_CB_STATUS_END }; - /* for preload and one shot copy just start the DMA and wait */ - if (flags & (DMA_COPY_PRELOAD | DMA_COPY_ONE_SHOT)) { + tracev_dwdma("dw_dma_copy(): dma %d channel %d copy", + channel->dma->plat_data.id, channel->index); + + if (channel->cb && channel->cb_type & DMA_CB_TYPE_COPY) + channel->cb(channel->cb_data, DMA_CB_TYPE_COPY, &next); + + if (flags & DMA_COPY_ONE_SHOT) { + /* for one shot copy start the DMA */ ret = dw_dma_start(channel); if (ret < 0) return ret; + } + if (flags & DMA_COPY_BLOCKING) { + /* wait for transfer finish */ ret = poll_for_register_delay(dma_base(channel->dma) + DW_DMA_CHAN_EN, DW_CHAN(channel->index), 0, @@ -854,12 +870,6 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, return ret; } - tracev_dwdma("dw_dma_copy(): dma %d channel %d copy", - channel->dma->plat_data.id, channel->index); - - if (channel->cb && channel->cb_type & DMA_CB_TYPE_COPY) - channel->cb(channel->cb_data, DMA_CB_TYPE_COPY, &next); - dw_dma_verify_transfer(channel, &next); /* increment current pointer */ @@ -1078,6 +1088,12 @@ static int dw_dma_get_attribute(struct dma *dma, uint32_t type, case DMA_ATTR_COPY_ALIGNMENT: *value = DW_DMA_COPY_ALIGNMENT; break; + case DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT: + *value = PLATFORM_DCACHE_ALIGN; + break; + case DMA_ATTR_BUFFER_PERIOD_COUNT: + *value = DW_DMA_BUFFER_PERIOD_COUNT; + break; default: ret = -EINVAL; break; diff --git a/src/drivers/imx/edma.c b/src/drivers/imx/edma.c index 3acec254cdf4..2b3849d115a6 100644 --- a/src/drivers/imx/edma.c +++ b/src/drivers/imx/edma.c @@ -471,10 +471,17 @@ static int edma_get_attribute(struct dma *dma, uint32_t type, uint32_t *value) * size of the elementary copy). */ *value = 4; - return 0; + break; + case DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT: + *value = PLATFORM_DCACHE_ALIGN; + break; + case DMA_ATTR_BUFFER_PERIOD_COUNT: + *value = EDMA_BUFFER_PERIOD_COUNT; + break; default: return -ENOENT; /* Attribute not found */ } + return 0; } static int edma_get_data_size(struct dma_chan_data *channel, diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index 6e4eb015e4f6..0db6a0ca19e0 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -65,6 +65,10 @@ /* DGMBS align value */ #define HDA_DMA_BUFFER_ALIGNMENT 0x20 #define HDA_DMA_COPY_ALIGNMENT 0x20 +#define HDA_DMA_BUFFER_ADDRESS_ALIGNMENT 0x80 + +/* DMA number of buffer periods */ +#define HDA_DMA_BUFFER_PERIOD_COUNT 2 /* * DMA Pointer Trace @@ -270,10 +274,22 @@ static void hda_dma_post_copy(struct dma_chan_data *chan, int bytes) if (chan->cb) chan->cb(chan->cb_data, DMA_CB_TYPE_COPY, &next); - /* Force Host DMA to exit L1 */ if (chan->direction == DMA_DIR_HMEM_TO_LMEM || - chan->direction == DMA_DIR_LMEM_TO_HMEM) + chan->direction == DMA_DIR_LMEM_TO_HMEM) { + /* set BFPI to let host gateway know we have read size, + * which will trigger next copy start. + */ + hda_dma_inc_fp(chan, bytes); + + /* Force Host DMA to exit L1 */ pm_runtime_put(PM_RUNTIME_HOST_DMA_L1, 0); + } else { + /* + * set BFPI to let link gateway know we have read size, + * which will trigger next copy start. + */ + hda_dma_inc_link_fp(chan, bytes); + } } static int hda_dma_link_copy_ch(struct dma_chan_data *chan, int bytes) @@ -291,11 +307,6 @@ static int hda_dma_link_copy_ch(struct dma_chan_data *chan, int bytes) if (dgcs & DGCS_BOR) dma_chan_reg_update_bits(chan, DGCS, DGCS_BOR, DGCS_BOR); - /* - * set BFPI to let link gateway know we have read size, - * which will trigger next copy start. - */ - hda_dma_inc_link_fp(chan, bytes); hda_dma_post_copy(chan, bytes); hda_dma_get_dbg_vals(chan, HDA_DBG_POST, HDA_DBG_LINK); @@ -356,20 +367,6 @@ static int hda_dma_host_copy(struct dma_chan_data *channel, int bytes, hda_dma_get_dbg_vals(channel, HDA_DBG_PRE, HDA_DBG_HOST); - if (flags & DMA_COPY_PRELOAD) { - /* report lack of data if preload is not yet finished */ - ret = channel->direction == DMA_DIR_HMEM_TO_LMEM ? - hda_dma_is_buffer_full(channel) : - hda_dma_is_buffer_empty(channel); - if (!ret) - return -ENODATA; - } else { - /* set BFPI to let host gateway know we have read size, - * which will trigger next copy start. - */ - hda_dma_inc_fp(channel, bytes); - } - /* blocking mode copy */ if (flags & DMA_COPY_BLOCKING) { ret = channel->direction == DMA_DIR_HMEM_TO_LMEM ? @@ -606,6 +603,7 @@ static int hda_dma_set_config(struct dma_chan_data *channel, channel->direction = config->direction; channel->desc_count = config->elem_array.count; channel->is_scheduling_source = config->is_scheduling_source; + channel->period = config->period; /* validate - HDA only supports continuous elems of same size */ for (i = 0; i < config->elem_array.count; i++) { @@ -875,6 +873,12 @@ static int hda_dma_get_attribute(struct dma *dma, uint32_t type, case DMA_ATTR_COPY_ALIGNMENT: *value = HDA_DMA_COPY_ALIGNMENT; break; + case DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT: + *value = HDA_DMA_BUFFER_ADDRESS_ALIGNMENT; + break; + case DMA_ATTR_BUFFER_PERIOD_COUNT: + *value = HDA_DMA_BUFFER_PERIOD_COUNT; + break; default: ret = -EINVAL; break; diff --git a/src/drivers/intel/cavs/idc.c b/src/drivers/intel/cavs/idc.c index 7ac49b233248..e030b1cffaeb 100644 --- a/src/drivers/intel/cavs/idc.c +++ b/src/drivers/intel/cavs/idc.c @@ -168,15 +168,13 @@ static int idc_pipeline_trigger(uint32_t cmd) struct ipc *ipc = cache_to_uncache(_ipc); struct sof_ipc_stream *data = ipc->comp_data; struct ipc_comp_dev *pcm_dev; - struct edf_task_pdata *edf_pdata; - struct task *task; int ret; /* invalidate stream data */ dcache_invalidate_region(data, sizeof(*data)); /* check whether component exists */ - pcm_dev = ipc_get_comp(ipc, data->comp_id); + pcm_dev = ipc_get_comp_by_id(ipc, data->comp_id); if (!pcm_dev) return -ENODEV; @@ -187,17 +185,6 @@ static int idc_pipeline_trigger(uint32_t cmd) pipeline_cache(pcm_dev->cd->pipeline, pcm_dev->cd, CACHE_INVALIDATE); - - /* TODO: Temporary W/A for task sharing between cores. - * Should be removed after changing memory management for - * slave cores. - */ - if (pcm_dev->cd->pipeline->preload_task) { - task = pcm_dev->cd->pipeline->preload_task; - edf_pdata = edf_sch_get_pdata(task); - dcache_invalidate_region(edf_pdata, sizeof(*edf_pdata)); - task_context_cache(edf_pdata->ctx, CACHE_INVALIDATE); - } } /* check whether we are executing from the right core */ @@ -208,22 +195,9 @@ static int idc_pipeline_trigger(uint32_t cmd) ret = pipeline_trigger(pcm_dev->cd->pipeline, pcm_dev->cd, cmd); /* writeback pipeline on stop */ - if (cmd == COMP_TRIGGER_STOP) { - /* TODO: Temporary W/A for task sharing between cores. - * Should be removed after changing memory management for - * slave cores. - */ - if (pcm_dev->cd->pipeline->preload_task) { - task = pcm_dev->cd->pipeline->preload_task; - edf_pdata = edf_sch_get_pdata(task); - task_context_cache(edf_pdata->ctx, CACHE_WRITEBACK_INV); - dcache_writeback_invalidate_region(edf_pdata, - sizeof(*edf_pdata)); - } - + if (cmd == COMP_TRIGGER_STOP) pipeline_cache(pcm_dev->cd->pipeline, pcm_dev->cd, CACHE_WRITEBACK_INV); - } return ret; } @@ -246,7 +220,7 @@ static int idc_component_command(uint32_t cmd) data->rhdr.hdr.size - sizeof(*data)); /* check whether component exists */ - comp_dev = ipc_get_comp(ipc, data->comp_id); + comp_dev = ipc_get_comp_by_id(ipc, data->comp_id); if (!comp_dev) return -ENODEV; diff --git a/src/include/sof/audio/buffer.h b/src/include/sof/audio/buffer.h index 3dd92792f9e9..d7892fadcd69 100644 --- a/src/include/sof/audio/buffer.h +++ b/src/include/sof/audio/buffer.h @@ -48,8 +48,10 @@ struct comp_buffer { void *addr; /* buffer base address */ void *end_addr; /* buffer end address */ - /* IPC configuration */ - struct sof_ipc_buffer ipc_buffer; + /* configuration */ + uint32_t id; + uint32_t pipeline_id; + uint32_t caps; /* connected components */ struct comp_dev *source; /* source component */ @@ -114,6 +116,7 @@ struct comp_buffer { typedef void (*cache_buff_op)(struct comp_buffer *); /* pipeline buffer creation and destruction */ +struct comp_buffer *buffer_alloc(uint32_t size, uint32_t caps, uint32_t align); struct comp_buffer *buffer_new(struct sof_ipc_buffer *desc); int buffer_set_size(struct comp_buffer *buffer, uint32_t size); void buffer_free(struct comp_buffer *buffer); @@ -129,7 +132,7 @@ static inline void buffer_zero(struct comp_buffer *buffer) tracev_buffer("buffer_zero()"); bzero(buffer->addr, buffer->size); - if (buffer->ipc_buffer.caps & SOF_MEM_CAPS_DMA) + if (buffer->caps & SOF_MEM_CAPS_DMA) dcache_writeback_region(buffer->addr, buffer->size); } @@ -211,10 +214,12 @@ static inline void *buffer_get_frag(struct comp_buffer *buffer, void *ptr, return current; } -static inline void buffer_init(struct comp_buffer *buffer, uint32_t size) +static inline void buffer_init(struct comp_buffer *buffer, uint32_t size, + uint32_t caps) { buffer->alloc_size = size; buffer->size = size; + buffer->caps = caps; buffer->w_ptr = buffer->addr; buffer->r_ptr = buffer->addr; buffer->end_addr = buffer->addr + size; @@ -222,4 +227,41 @@ static inline void buffer_init(struct comp_buffer *buffer, uint32_t size) buffer->avail = 0; buffer_zero(buffer); } + +static inline void buffer_copy_s16(struct comp_buffer *source, + struct comp_buffer *sink, uint32_t bytes) +{ + uint32_t frames = bytes / sizeof(int16_t); + uint32_t buff_frag = 0; + int16_t *src; + int16_t *dst; + uint32_t i; + + for (i = 0; i < frames; i++) { + src = buffer_read_frag_s16(source, buff_frag); + dst = buffer_write_frag_s16(sink, buff_frag); + *dst = *src; + + buff_frag++; + } +} + +static inline void buffer_copy_s32(struct comp_buffer *source, + struct comp_buffer *sink, uint32_t bytes) +{ + uint32_t frames = bytes / sizeof(int32_t); + uint32_t buff_frag = 0; + int32_t *src; + int32_t *dst; + uint32_t i; + + for (i = 0; i < frames; i++) { + src = buffer_read_frag_s32(source, buff_frag); + dst = buffer_write_frag_s32(sink, buff_frag); + *dst = *src; + + buff_frag++; + } +} + #endif /* __SOF_AUDIO_BUFFER_H__ */ diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 47d5265e108f..eb21cd666d62 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -233,7 +233,6 @@ struct comp_dev { /* runtime */ uint16_t state; /**< COMP_STATE_ */ - uint16_t is_dma_connected; /**< component is connected to DMA */ uint64_t position; /**< component rendering position */ uint32_t frames; /**< number of frames we copy to sink */ uint32_t output_rate; /**< 0 means all output rates are fine */ @@ -379,15 +378,6 @@ static inline void comp_free(struct comp_dev *dev) */ int comp_set_state(struct comp_dev *dev, int cmd); -/** - * Set component's sink buffer size - * @param dev Component device. - * @param period_bytes Amount of bytes in one period. - * @param periods Amount of periods. - */ -int comp_set_sink_buffer(struct comp_dev *dev, uint32_t period_bytes, - uint32_t periods); - /** * Component parameter init. * @param dev Component device. diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 82e03115cc51..807cba4390e4 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -75,11 +75,9 @@ struct pipeline { /* runtime status */ int32_t xrun_bytes; /* last xrun length */ uint32_t status; /* pipeline status */ - bool preload; /* is pipeline preload needed */ /* scheduling */ struct task *pipe_task; /* pipeline processing task */ - struct task *preload_task; /* pipeline preload task */ /* component that drives scheduling in this pipe */ struct comp_dev *sched_comp; @@ -102,12 +100,6 @@ static inline bool pipeline_is_same_sched_comp(struct pipeline *current, return current->sched_comp == previous->sched_comp; } -/* checks if pipeline is in preload phase */ -static inline bool pipeline_is_preload(struct pipeline *p) -{ - return p->preload; -} - /* checks if pipeline is scheduled with timer */ static inline bool pipeline_is_timer_driven(struct pipeline *p) { diff --git a/src/include/sof/drivers/dw-dma.h b/src/include/sof/drivers/dw-dma.h index aeb1a7aac394..8667a14679ca 100644 --- a/src/include/sof/drivers/dw-dma.h +++ b/src/include/sof/drivers/dw-dma.h @@ -134,8 +134,8 @@ #define trace_dwdma_error(__e, ...) \ trace_error(TRACE_CLASS_DMA, __e, ##__VA_ARGS__) -#define DW_DMA_BUFFER_ALIGNMENT 0x20 -#define DW_DMA_COPY_ALIGNMENT 0x20 +#define DW_DMA_BUFFER_ALIGNMENT 0x4 +#define DW_DMA_COPY_ALIGNMENT 0x4 /* TODO: add FIFO sizes */ struct dw_chan_data { diff --git a/src/include/sof/drivers/edma.h b/src/include/sof/drivers/edma.h index 6adce4a0e6ae..0975c079bac4 100644 --- a/src/include/sof/drivers/edma.h +++ b/src/include/sof/drivers/edma.h @@ -75,6 +75,8 @@ int edma_init(void); +#define EDMA_BUFFER_PERIOD_COUNT 2 + #define EDMA_TCD_ALIGNMENT 32 #define EDMA_HS_GET_IRQ(hs) (((hs) & MASK(8, 0)) >> 0) diff --git a/src/include/sof/drivers/ipc.h b/src/include/sof/drivers/ipc.h index 43fb528298f4..af381381017b 100644 --- a/src/include/sof/drivers/ipc.h +++ b/src/include/sof/drivers/ipc.h @@ -224,7 +224,13 @@ int ipc_comp_connect(struct ipc *ipc, /* * Get component by ID. */ -struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id); +struct ipc_comp_dev *ipc_get_comp_by_id(struct ipc *ipc, uint32_t id); + +/* + * Get component by pipeline ID. + */ +struct ipc_comp_dev *ipc_get_comp_by_ppl_id(struct ipc *ipc, uint16_t type, + uint32_t ppl_id); /* * Configure all DAI components attached to DAI. diff --git a/src/include/sof/lib/alloc.h b/src/include/sof/lib/alloc.h index 00a8cae11baf..3f7c4d1c476b 100644 --- a/src/include/sof/lib/alloc.h +++ b/src/include/sof/lib/alloc.h @@ -62,6 +62,7 @@ struct mm_info { struct block_hdr { uint16_t size; /* size in blocks for continuous allocation */ uint16_t used; /* usage flags for page */ + void *unaligned_ptr; /* align ptr */ } __packed; struct block_map { @@ -105,9 +106,10 @@ struct mm { /* heap allocation and free */ void *_malloc(int zone, uint32_t caps, size_t bytes); void *_zalloc(int zone, uint32_t caps, size_t bytes); -void *_balloc(int zone, uint32_t caps, size_t bytes); +void *_balloc(int zone, uint32_t caps, size_t bytes, uint32_t alignment); void *_realloc(void *ptr, int zone, uint32_t caps, size_t bytes); -void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes); +void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes, + uint32_t alignment); void rfree(void *ptr); #if CONFIG_DEBUG_HEAP @@ -143,7 +145,7 @@ void rfree(void *ptr); #define rballoc(zone, caps, bytes) \ ({void *_ptr; \ do { \ - _ptr = _balloc(zone, caps, bytes); \ + _ptr = _balloc(zone, caps, bytes, PLATFORM_DCACHE_ALIGN);\ if (!_ptr) { \ trace_error(TRACE_CLASS_MEM, \ "failed to alloc 0x%x bytes caps 0x%x", \ @@ -169,7 +171,34 @@ void rfree(void *ptr); #define rbrealloc(ptr, zone, caps, bytes) \ ({void *_ptr; \ do { \ - _ptr = _brealloc(ptr, zone, caps, bytes); \ + _ptr = _brealloc(ptr, zone, caps, bytes,\ + PLATFORM_DCACHE_ALIGN);\ + if (!_ptr) { \ + trace_error(TRACE_CLASS_MEM, \ + "failed to alloc 0x%x bytes caps 0x%x", \ + bytes, caps); \ + alloc_trace_buffer_heap(zone, caps, bytes); \ + } \ + } while (0); \ + _ptr; }) + +#define rbrealloc_align(ptr, zone, caps, bytes, alignment) \ + ({void *_ptr; \ + do { \ + _ptr = _brealloc(ptr, zone, caps, bytes, alignment); \ + if (!_ptr) { \ + trace_error(TRACE_CLASS_MEM, \ + "failed to alloc 0x%x bytes caps 0x%x", \ + bytes, caps); \ + alloc_trace_buffer_heap(zone, caps, bytes); \ + } \ + } while (0); \ + _ptr; }) + +#define rballoc(zone, caps, bytes, alignment) \ + ({void *_ptr; \ + do { \ + _ptr = _balloc(zone, caps, bytes, alignment); \ if (!_ptr) { \ trace_error(TRACE_CLASS_MEM, \ "failed to alloc 0x%x bytes caps 0x%x", \ @@ -186,11 +215,16 @@ void alloc_trace_buffer_heap(int zone, uint32_t caps, size_t bytes); #define rmalloc(zone, caps, bytes) _malloc(zone, caps, bytes) #define rzalloc(zone, caps, bytes) _zalloc(zone, caps, bytes) -#define rballoc(zone, caps, bytes) _balloc(zone, caps, bytes) +#define rballoc(zone, caps, bytes) \ + _balloc(zone, caps, bytes, PLATFORM_DCACHE_ALIGN) +#define rballoc_align(zone, caps, bytes, alignment) \ + _balloc(zone, caps, bytes, alignment) #define rrealloc(ptr, zone, caps, bytes) \ _realloc(ptr, zone, caps, bytes) #define rbrealloc(ptr, zone, caps, bytes) \ - _brealloc(ptr, zone, caps, bytes) + _brealloc(ptr, zone, caps, bytes, PLATFORM_DCACHE_ALIGN) +#define rbrealloc_align(ptr, zone, caps, bytes, alignment) \ + _brealloc(ptr, zone, caps, bytes, alignment) #endif diff --git a/src/include/sof/lib/cache.h b/src/include/sof/lib/cache.h index 6144abeb7a7d..1418845fc096 100644 --- a/src/include/sof/lib/cache.h +++ b/src/include/sof/lib/cache.h @@ -22,11 +22,4 @@ /* invalidate data */ #define CACHE_INVALIDATE 1 -/* data cache line alignment */ -#if DCACHE_LINE_SIZE > 0 -#define PLATFORM_DCACHE_ALIGN DCACHE_LINE_SIZE -#else -#define PLATFORM_DCACHE_ALIGN sizeof(void *) -#endif - #endif /* __SOF_LIB_CACHE_H__ */ diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index 5a21134dfc79..fe86c54d7211 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -26,6 +26,8 @@ #include #include +struct comp_buffer; + /** \addtogroup sof_dma_drivers DMA Drivers * DMA Drivers API specification. * @{ @@ -64,9 +66,8 @@ #define DMA_CB_TYPE_COPY BIT(1) /* DMA copy flags */ -#define DMA_COPY_PRELOAD BIT(0) -#define DMA_COPY_BLOCKING BIT(1) -#define DMA_COPY_ONE_SHOT BIT(2) +#define DMA_COPY_BLOCKING BIT(0) +#define DMA_COPY_ONE_SHOT BIT(1) /* We will use this enum in cb handler to inform dma what * action we need to perform. @@ -88,8 +89,10 @@ enum dma_irq_cmd { #define DMA_CORE_INVALID 0xFFFFFFFF /* DMA attributes */ -#define DMA_ATTR_BUFFER_ALIGNMENT 0 -#define DMA_ATTR_COPY_ALIGNMENT 1 +#define DMA_ATTR_BUFFER_ALIGNMENT 0 +#define DMA_ATTR_COPY_ALIGNMENT 1 +#define DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT 2 +#define DMA_ATTR_BUFFER_PERIOD_COUNT 3 struct dma; @@ -127,6 +130,7 @@ struct dma_sg_config { uint32_t src_dev; uint32_t dest_dev; uint32_t cyclic; /* circular buffer */ + uint64_t period; struct dma_sg_elem_array elem_array; /* array of dma_sg elems */ bool scatter; bool irq_disabled; @@ -210,6 +214,7 @@ struct dma_chan_data { uint32_t desc_count; uint32_t index; uint32_t core; + uint64_t period; /* DMA channel's transfer period */ /* true if this DMA channel is the scheduling source */ bool is_scheduling_source; @@ -498,6 +503,16 @@ static inline uint32_t dma_sg_get_size(struct dma_sg_elem_array *ea) return size; } +/* copies data from DMA buffer using provided processing function */ +void dma_buffer_copy_from(struct comp_buffer *source, struct comp_buffer *sink, + void (*process)(struct comp_buffer *, struct comp_buffer *, uint32_t), + uint32_t bytes); + +/* copies data to DMA buffer using provided processing function */ +void dma_buffer_copy_to(struct comp_buffer *source, struct comp_buffer *sink, + void (*process)(struct comp_buffer *, struct comp_buffer *, uint32_t), + uint32_t bytes); + /* generic DMA DSP <-> Host copier */ struct dma_copy { diff --git a/src/include/sof/lib/notifier.h b/src/include/sof/lib/notifier.h index da3ee3a6a507..8301b1ddd46e 100644 --- a/src/include/sof/lib/notifier.h +++ b/src/include/sof/lib/notifier.h @@ -22,6 +22,7 @@ enum notify_id { NOTIFIER_ID_CPU_FREQ = 0, NOTIFIER_ID_SSP_FREQ, NOTIFIER_ID_KPB_CLIENT_EVT, + NOTIFIER_ID_DMA_DOMAIN_CHANGE, }; struct notify { diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index dc5e0a43c02a..1c352bf23b6a 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -131,4 +131,8 @@ struct ll_schedule_domain *dma_multi_chan_domain_init(struct dma *dma_array, uint32_t num_dma, int clk, bool aggregated_irq); +struct ll_schedule_domain *dma_single_chan_domain_init(struct dma *dma_array, + uint32_t num_dma, + int clk); + #endif /* __SOF_SCHEDULE_LL_SCHEDULE_DOMAIN_H__ */ diff --git a/src/ipc/dma-copy.c b/src/ipc/dma-copy.c index c6f058c0c2a0..619e7919fffe 100644 --- a/src/ipc/dma-copy.c +++ b/src/ipc/dma-copy.c @@ -112,7 +112,8 @@ int dma_copy_to_host_nowait(struct dma_copy *dc, struct dma_sg_config *host_sg, if (err < 0) return err; - err = dma_copy(dc->chan, local_sg_elem.size, DMA_COPY_ONE_SHOT); + err = dma_copy(dc->chan, local_sg_elem.size, + DMA_COPY_ONE_SHOT | DMA_COPY_BLOCKING); if (err < 0) return err; diff --git a/src/ipc/handler.c b/src/ipc/handler.c index 1e1580b9e24c..8aa14b936b32 100644 --- a/src/ipc/handler.c +++ b/src/ipc/handler.c @@ -224,7 +224,7 @@ static int ipc_stream_pcm_params(uint32_t stream) trace_ipc("ipc: comp %d -> params", pcm_params.comp_id); /* get the pcm_dev */ - pcm_dev = ipc_get_comp(_ipc, pcm_params.comp_id); + pcm_dev = ipc_get_comp_by_id(_ipc, pcm_params.comp_id); if (pcm_dev == NULL) { trace_ipc_error("ipc: comp %d not found", pcm_params.comp_id); return -ENODEV; @@ -344,7 +344,7 @@ static int ipc_stream_pcm_free(uint32_t header) trace_ipc("ipc: comp %d -> free", free_req.comp_id); /* get the pcm_dev */ - pcm_dev = ipc_get_comp(_ipc, free_req.comp_id); + pcm_dev = ipc_get_comp_by_id(_ipc, free_req.comp_id); if (pcm_dev == NULL) { trace_ipc_error("ipc: comp %d not found", free_req.comp_id); return -ENODEV; @@ -376,7 +376,7 @@ static int ipc_stream_position(uint32_t header) memset(&posn, 0, sizeof(posn)); /* get the pcm_dev */ - pcm_dev = ipc_get_comp(_ipc, stream.comp_id); + pcm_dev = ipc_get_comp_by_id(_ipc, stream.comp_id); if (pcm_dev == NULL) { trace_ipc_error("ipc: comp %d not found", stream.comp_id); return -ENODEV; @@ -455,7 +455,7 @@ static int ipc_stream_trigger(uint32_t header) trace_ipc("ipc: comp %d -> trigger cmd 0x%x", stream.comp_id, ipc_cmd); /* get the pcm_dev */ - pcm_dev = ipc_get_comp(_ipc, stream.comp_id); + pcm_dev = ipc_get_comp_by_id(_ipc, stream.comp_id); if (pcm_dev == NULL) { trace_ipc_error("ipc: comp %d not found", stream.comp_id); return -ENODEV; @@ -885,7 +885,7 @@ static int ipc_comp_value(uint32_t header, uint32_t cmd) trace_ipc("ipc: comp %d -> cmd %d", data.comp_id, data.cmd); /* get the component */ - comp_dev = ipc_get_comp(_ipc, data.comp_id); + comp_dev = ipc_get_comp_by_id(_ipc, data.comp_id); if (!comp_dev) { trace_ipc_error("ipc: comp %d not found", data.comp_id); return -ENODEV; diff --git a/src/ipc/ipc-host-ptable.c b/src/ipc/ipc-host-ptable.c index 7b30cff8bca6..1b7c8b490c9a 100644 --- a/src/ipc/ipc-host-ptable.c +++ b/src/ipc/ipc-host-ptable.c @@ -120,7 +120,7 @@ static int ipc_get_page_descriptors(struct dma *dmac, uint8_t *page_table, } /* start the copy of page table to DSP */ - ret = dma_copy(chan, elem.size, DMA_COPY_ONE_SHOT); + ret = dma_copy(chan, elem.size, DMA_COPY_ONE_SHOT | DMA_COPY_BLOCKING); if (ret < 0) { trace_ipc_error("ipc_get_page_descriptors() error: " "dma_start() failed"); diff --git a/src/ipc/ipc.c b/src/ipc/ipc.c index 908011daa083..d709ad0673e4 100644 --- a/src/ipc/ipc.c +++ b/src/ipc/ipc.c @@ -38,7 +38,7 @@ * more than 1 list may need to be searched for the corresponding component. */ -struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id) +struct ipc_comp_dev *ipc_get_comp_by_id(struct ipc *ipc, uint32_t id) { struct ipc_comp_dev *icd; struct list_item *clist; @@ -51,7 +51,7 @@ struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id) return icd; break; case COMP_TYPE_BUFFER: - if (icd->cb->ipc_buffer.comp.id == id) + if (icd->cb->id == id) return icd; break; case COMP_TYPE_PIPELINE: @@ -66,6 +66,36 @@ struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id) return NULL; } +struct ipc_comp_dev *ipc_get_comp_by_ppl_id(struct ipc *ipc, uint16_t type, + uint32_t ppl_id) +{ + struct ipc_comp_dev *icd; + struct list_item *clist; + + list_for_item(clist, &ipc->shared_ctx->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->type != type) + continue; + + switch (icd->type) { + case COMP_TYPE_COMPONENT: + if (icd->cd->comp.pipeline_id == ppl_id) + return icd; + break; + case COMP_TYPE_BUFFER: + if (icd->cb->pipeline_id == ppl_id) + return icd; + break; + case COMP_TYPE_PIPELINE: + if (icd->pipeline->ipc_pipe.pipeline_id == ppl_id) + return icd; + break; + } + } + + return NULL; +} + static struct ipc_comp_dev *ipc_get_ppl_comp(struct ipc *ipc, uint32_t pipeline_id, int dir) { @@ -129,7 +159,7 @@ int ipc_comp_new(struct ipc *ipc, struct sof_ipc_comp *comp) int ret = 0; /* check whether component already exists */ - icd = ipc_get_comp(ipc, comp->id); + icd = ipc_get_comp_by_id(ipc, comp->id); if (icd != NULL) { trace_ipc_error("ipc_comp_new() error: comp->id = %u", comp->id); @@ -164,7 +194,7 @@ int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) struct ipc_comp_dev *icd; /* check whether component exists */ - icd = ipc_get_comp(ipc, comp_id); + icd = ipc_get_comp_by_id(ipc, comp_id); if (icd == NULL) return -ENODEV; @@ -196,7 +226,7 @@ int ipc_buffer_new(struct ipc *ipc, struct sof_ipc_buffer *desc) int ret = 0; /* check whether buffer already exists */ - ibd = ipc_get_comp(ipc, desc->comp.id); + ibd = ipc_get_comp_by_id(ipc, desc->comp.id); if (ibd != NULL) { trace_ipc_error("ipc_buffer_new() error: " "buffer already exists, desc->comp.id = %u", @@ -231,7 +261,7 @@ int ipc_buffer_free(struct ipc *ipc, uint32_t buffer_id) struct ipc_comp_dev *ibd; /* check whether buffer exists */ - ibd = ipc_get_comp(ipc, buffer_id); + ibd = ipc_get_comp_by_id(ipc, buffer_id); if (ibd == NULL) return -ENODEV; @@ -250,7 +280,7 @@ int ipc_comp_connect(struct ipc *ipc, struct ipc_comp_dev *icd_sink; /* check whether the components already exist */ - icd_source = ipc_get_comp(ipc, connect->source_id); + icd_source = ipc_get_comp_by_id(ipc, connect->source_id); if (icd_source == NULL) { trace_ipc_error("ipc_comp_connect() error: components already" " exist, connect->source_id = %u", @@ -258,7 +288,7 @@ int ipc_comp_connect(struct ipc *ipc, return -EINVAL; } - icd_sink = ipc_get_comp(ipc, connect->sink_id); + icd_sink = ipc_get_comp_by_id(ipc, connect->sink_id); if (icd_sink == NULL) { trace_ipc_error("ipc_comp_connect() error: components already" " exist, connect->sink_id = %u", @@ -293,7 +323,7 @@ int ipc_pipeline_new(struct ipc *ipc, struct ipc_comp_dev *icd; /* check whether the pipeline already exists */ - ipc_pipe = ipc_get_comp(ipc, pipe_desc->comp_id); + ipc_pipe = ipc_get_comp_by_id(ipc, pipe_desc->comp_id); if (ipc_pipe != NULL) { trace_ipc_error("ipc_pipeline_new() error: pipeline already" " exists, pipe_desc->comp_id = %u", @@ -301,8 +331,18 @@ int ipc_pipeline_new(struct ipc *ipc, return -EINVAL; } + /* check whether pipeline id is already taken */ + ipc_pipe = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + pipe_desc->pipeline_id); + if (ipc_pipe) { + trace_ipc_error("ipc_pipeline_new() error: pipeline id is " + "already taken, pipe_desc->pipeline_id = %u", + pipe_desc->pipeline_id); + return -EINVAL; + } + /* find the scheduling component */ - icd = ipc_get_comp(ipc, pipe_desc->sched_id); + icd = ipc_get_comp_by_id(ipc, pipe_desc->sched_id); if (icd == NULL) { trace_ipc_error("ipc_pipeline_new() error: cannot find the " "scheduling component, pipe_desc->sched_id" @@ -345,7 +385,7 @@ int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) int ret; /* check whether pipeline exists */ - ipc_pipe = ipc_get_comp(ipc, comp_id); + ipc_pipe = ipc_get_comp_by_id(ipc, comp_id); if (ipc_pipe == NULL) return -ENODEV; @@ -371,7 +411,7 @@ int ipc_pipeline_complete(struct ipc *ipc, uint32_t comp_id) struct ipc_comp_dev *ipc_ppl_sink; /* check whether pipeline exists */ - ipc_pipe = ipc_get_comp(ipc, comp_id); + ipc_pipe = ipc_get_comp_by_id(ipc, comp_id); if (!ipc_pipe) return -EINVAL; @@ -397,7 +437,7 @@ int ipc_comp_dai_config(struct ipc *ipc, struct sof_ipc_dai_config *config) struct ipc_comp_dev *icd; struct list_item *clist; struct comp_dev *dev; - int ret = 0; + int ret = -ENODEV; /* for each component */ list_for_item(clist, &ipc->shared_ctx->comp_list) { diff --git a/src/lib/alloc.c b/src/lib/alloc.c index 2a9d7b3db720..84d785e78077 100644 --- a/src/lib/alloc.c +++ b/src/lib/alloc.c @@ -201,9 +201,30 @@ static void *rmalloc_sys(int zone, int caps, int core, size_t bytes) return ptr; } +/* At this point the pointer we have should be unaligned + * (it was checked level higher) and be power of 2 + */ +static void *align_ptr(struct mm_heap *heap, uint32_t alignment, + void *ptr, struct block_hdr *hdr) +{ + int mod_align = 0; + + /* Save unaligned ptr to block hdr */ + hdr->unaligned_ptr = ptr; + + /* If ptr is not already aligned we calculate alignment shift */ + if (alignment && (uintptr_t)ptr % alignment) + mod_align = alignment - ((uintptr_t)ptr % alignment); + + /* Calculate aligned pointer */ + ptr = ptr + mod_align; + + return ptr; +} + /* allocate single block */ static void *alloc_block(struct mm_heap *heap, int level, - uint32_t caps) + uint32_t caps, uint32_t alignment) { struct block_map *map = &heap->map[level]; struct block_hdr *hdr; @@ -216,8 +237,11 @@ static void *alloc_block(struct mm_heap *heap, int level, map->free_count--; ptr = (void *)(map->base + map->first_free * map->block_size); + ptr = align_ptr(heap, alignment, ptr, hdr); + hdr->size = 1; hdr->used = 1; + heap->info.used += map->block_size; heap->info.free -= map->block_size; @@ -239,11 +263,11 @@ static void *alloc_block(struct mm_heap *heap, int level, /* allocates continuous blocks */ static void *alloc_cont_blocks(struct mm_heap *heap, int level, - uint32_t caps, size_t bytes) + uint32_t caps, size_t bytes, uint32_t alignment) { struct block_map *map = &heap->map[level]; struct block_hdr *hdr; - void *ptr; + void *ptr, *unaligned_ptr; unsigned int start = map->first_free; unsigned int current; unsigned int count = bytes / map->block_size; @@ -277,8 +301,13 @@ static void *alloc_cont_blocks(struct mm_heap *heap, int level, /* we found enough space, let's allocate it */ map->free_count -= count; ptr = (void *)(map->base + start * map->block_size); + unaligned_ptr = ptr; + hdr = &map->block[start]; hdr->size = count; + + ptr = align_ptr(heap, alignment, ptr, hdr); + heap->info.used += count * map->block_size; heap->info.free -= count * map->block_size; /* update first_free if needed */ @@ -296,6 +325,7 @@ static void *alloc_cont_blocks(struct mm_heap *heap, int level, for (current = start; current < start + count; current++) { hdr = &map->block[current]; hdr->used = 1; + hdr->unaligned_ptr = unaligned_ptr; } writeback_block_map(map); @@ -356,27 +386,43 @@ static struct mm_heap *get_heap_from_caps(struct mm_heap *heap, int count, } static void *get_ptr_from_heap(struct mm_heap *heap, int zone, uint32_t caps, - size_t bytes) + size_t bytes, uint32_t alignment) { struct block_map *map; - int i; + int i, temp_bytes = bytes; void *ptr = NULL; + /* Only allow alignment as a power of 2 */ + if ((alignment & (alignment - 1)) != 0) + panic(SOF_IPC_PANIC_MEM); + for (i = 0; i < heap->blocks; i++) { map = &heap->map[i]; dcache_invalidate_region(map, sizeof(*map)); + /* size of requested buffer is adjusted for alignment purposes + * we check if first free block is already aligned if not + * we need to allocate bigger size for alignment + */ + if ((map->base + (map->block_size * map->first_free)) + % alignment) + temp_bytes += alignment; + /* is block big enough */ - if (map->block_size < bytes) + if (map->block_size < temp_bytes) { + temp_bytes = bytes; continue; + } /* does block have free space */ - if (map->free_count == 0) + if (map->free_count == 0) { + temp_bytes = bytes; continue; + } /* free block space exists */ - ptr = alloc_block(heap, i, caps); + ptr = alloc_block(heap, i, caps, alignment); break; } @@ -428,14 +474,26 @@ static void free_block(void *ptr) /* calculate block header */ block = ((uint32_t)ptr - block_map->base) / block_map->block_size; - /* report an error if ptr is not aligned to block */ - if (block_map->base + block_map->block_size * block != (uint32_t)ptr) - panic(SOF_IPC_PANIC_MEM); - invalidate_blocks(block_map); hdr = &block_map->block[block]; + /* bring back original unaligned pointer position + * and calculate correct hdr for free operation (it could + * be from different block since we got user pointer here + * or null if header was not set) + */ + if (hdr->unaligned_ptr != ptr && hdr->unaligned_ptr) { + ptr = hdr->unaligned_ptr; + block = ((uint32_t)ptr - block_map->base) + / block_map->block_size; + hdr = &block_map->block[block]; + } + + /* report an error if ptr is not aligned to block */ + if (block_map->base + block_map->block_size * block != (uint32_t)ptr) + panic(SOF_IPC_PANIC_MEM); + /* free block header and continuous blocks */ used_blocks = block + hdr->size; @@ -443,6 +501,7 @@ static void free_block(void *ptr) hdr = &block_map->block[i]; hdr->size = 0; hdr->used = 0; + hdr->unaligned_ptr = NULL; block_map->free_count++; heap->info.used -= block_map->block_size; heap->info.free += block_map->block_size; @@ -547,7 +606,8 @@ static void *rmalloc_sys_runtime(int zone, int caps, int core, size_t bytes) if ((cpu_heap->caps & caps) != caps) panic(SOF_IPC_PANIC_MEM); - ptr = get_ptr_from_heap(cpu_heap, zone, caps, bytes); + ptr = get_ptr_from_heap(cpu_heap, zone, caps, bytes, + PLATFORM_DCACHE_ALIGN); /* other core should have the latest value */ if (core != cpu_get_id()) @@ -577,8 +637,8 @@ static void *rmalloc_runtime(int zone, uint32_t caps, size_t bytes) return NULL; } } - - return get_ptr_from_heap(heap, zone, caps, bytes); + return get_ptr_from_heap(heap, zone, caps, bytes, + PLATFORM_DCACHE_ALIGN); } static void *_malloc_unlocked(int zone, uint32_t caps, size_t bytes) @@ -657,26 +717,45 @@ void *rzalloc_core_sys(int core, size_t bytes) /* allocates continuous buffers - not for direct use, clients use rballoc() */ static void *alloc_heap_buffer(struct mm_heap *heap, int zone, uint32_t caps, - size_t bytes) + size_t bytes, uint32_t alignment) { struct block_map *map; - int i; + int i, temp_bytes = bytes; void *ptr = NULL; + /* Only allow alignment as a power of 2 */ + if ((alignment & (alignment - 1)) != 0) + panic(SOF_IPC_PANIC_MEM); + /* will request fit in single block */ for (i = 0; i < heap->blocks; i++) { map = &heap->map[i]; dcache_invalidate_region(map, sizeof(*map)); + /* size of requested buffer is adjusted for alignment purposes + * we check if first free block is already aligned if not + * we need to allocate bigger size for alignment + */ + if ((map->base + (map->block_size * map->first_free)) + % alignment) + temp_bytes += alignment; + /* Check if blocks are big enough and at least one is free */ - if (map->block_size >= bytes && map->free_count) { + if (map->block_size >= temp_bytes && map->free_count) { /* found: grab a block */ - ptr = alloc_block(heap, i, caps); + + ptr = alloc_block(heap, i, caps, alignment); break; } + temp_bytes = bytes; } + /* size of requested buffer is adjusted for alignment purposes + * since we span more blocks we have to assume worst case scenario + */ + bytes += alignment; + /* request spans > 1 block */ if (!ptr) { /* @@ -689,8 +768,9 @@ static void *alloc_heap_buffer(struct mm_heap *heap, int zone, uint32_t caps, dcache_invalidate_region(map, sizeof(*map)); /* allocate if block size is smaller than request */ - if (heap->size >= bytes && map->block_size < bytes) { - ptr = alloc_cont_blocks(heap, i, caps, bytes); + if (heap->size >= bytes && map->block_size < bytes) { + ptr = alloc_cont_blocks(heap, i, caps, + bytes, alignment); if (ptr) break; } @@ -702,13 +782,14 @@ static void *alloc_heap_buffer(struct mm_heap *heap, int zone, uint32_t caps, #if CONFIG_DEBUG_BLOCK_FREE if (ptr) - bzero(ptr, bytes); + bzero(ptr, temp_bytes); #endif return ptr; } -static void *_balloc_unlocked(int zone, uint32_t caps, size_t bytes) +static void *_balloc_unlocked(int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { struct mm_heap *heap; unsigned int i, n; @@ -722,7 +803,7 @@ static void *_balloc_unlocked(int zone, uint32_t caps, size_t bytes) if (!heap) break; - ptr = alloc_heap_buffer(heap, zone, caps, bytes); + ptr = alloc_heap_buffer(heap, zone, caps, bytes, alignment); if (ptr) break; @@ -733,14 +814,14 @@ static void *_balloc_unlocked(int zone, uint32_t caps, size_t bytes) } /* allocates continuous buffers - not for direct use, clients use rballoc() */ -void *_balloc(int zone, uint32_t caps, size_t bytes) +void *_balloc(int zone, uint32_t caps, size_t bytes, uint32_t alignment) { void *ptr = NULL; uint32_t flags; spin_lock_irq(memmap.lock, flags); - ptr = _balloc_unlocked(zone, caps, bytes); + ptr = _balloc_unlocked(zone, caps, bytes, alignment); spin_unlock_irq(memmap.lock, flags); @@ -809,7 +890,8 @@ void *_realloc(void *ptr, int zone, uint32_t caps, size_t bytes) return new_ptr; } -void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes) +void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { void *new_ptr = NULL; uint32_t flags; @@ -819,7 +901,7 @@ void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes) spin_lock_irq(memmap.lock, flags); - new_ptr = _balloc_unlocked(zone, caps, bytes); + new_ptr = _balloc_unlocked(zone, caps, bytes, alignment); if (new_ptr && ptr) memcpy_s(new_ptr, bytes, ptr, bytes); diff --git a/src/lib/dma.c b/src/lib/dma.c index 510b612c55a2..a24da56a4905 100644 --- a/src/lib/dma.c +++ b/src/lib/dma.c @@ -5,7 +5,9 @@ // Author: Ranjani Sridharan #include +#include #include +#include #include #include #include @@ -186,3 +188,63 @@ void dma_sg_free(struct dma_sg_elem_array *elem_array) rfree(elem_array->elems); dma_sg_init(elem_array); } + +void dma_buffer_copy_from(struct comp_buffer *source, struct comp_buffer *sink, + void (*process)(struct comp_buffer *, struct comp_buffer *, uint32_t), + uint32_t bytes) +{ + uint32_t head = bytes; + uint32_t tail = 0; + + /* source buffer contains data copied by DMA */ + if (source->r_ptr + bytes > source->end_addr) { + head = source->end_addr - source->r_ptr; + tail = bytes - head; + } + + dcache_invalidate_region(source->r_ptr, head); + if (tail) + dcache_invalidate_region(source->addr, tail); + + /* process data */ + process(source, sink, bytes); + + source->r_ptr += bytes; + + /* check for pointer wrap */ + if (source->r_ptr >= source->end_addr) + source->r_ptr = source->addr + + (source->r_ptr - source->end_addr); + + comp_update_buffer_produce(sink, bytes); +} + +void dma_buffer_copy_to(struct comp_buffer *source, struct comp_buffer *sink, + void (*process)(struct comp_buffer *, struct comp_buffer *, uint32_t), + uint32_t bytes) +{ + uint32_t head = bytes; + uint32_t tail = 0; + + /* process data */ + process(source, sink, bytes); + + /* sink buffer contains data meant to copied to DMA */ + if (sink->w_ptr + bytes > sink->end_addr) { + head = sink->end_addr - sink->w_ptr; + tail = bytes - head; + } + + dcache_writeback_region(sink->w_ptr, head); + if (tail) + dcache_writeback_region(sink->addr, tail); + + sink->w_ptr += bytes; + + /* check for pointer wrap */ + if (sink->w_ptr >= sink->end_addr) + sink->w_ptr = sink->addr + + (sink->w_ptr - sink->end_addr); + + comp_update_buffer_consume(source, bytes); +} diff --git a/src/platform/apollolake/apollolake.x.in b/src/platform/apollolake/apollolake.x.in index e61bf37fb46f..c24ed758f4b8 100644 --- a/src/platform/apollolake/apollolake.x.in +++ b/src/platform/apollolake/apollolake.x.in @@ -89,9 +89,6 @@ MEMORY sof_fw : org = SOF_FW_BASE, len = SOF_FW_MAX_SIZE - buffer_hp_heap : - org = HEAP_HP_BUFFER_BASE, - len = HEAP_HP_BUFFER_SIZE wnd0 : org = HP_SRAM_WIN0_BASE, len = HP_SRAM_WIN0_SIZE @@ -136,7 +133,6 @@ PHDRS vector_double_lit_phdr PT_LOAD; vector_double_text_phdr PT_LOAD; sof_fw_phdr PT_LOAD; - buffer_hp_heap_phdr PT_LOAD; wnd0_phdr PT_LOAD; wnd1_phdr PT_LOAD; wnd2_phdr PT_LOAD; @@ -174,14 +170,6 @@ SECTIONS _MemoryExceptionVector_text_end = ABSOLUTE(.); } >vector_memory_text :vector_memory_text_phdr - .buffer_hp_heap (NOLOAD) : ALIGN(8) - { - . = ALIGN (32); - _buffer_hp_heap_start = ABSOLUTE(.); - . = . + HEAP_HP_BUFFER_SIZE; - _buffer_hp_heap_end = ABSOLUTE(.); - } >buffer_hp_heap :buffer_hp_heap_phdr - .wnd0 (NOLOAD) : ALIGN(8) { . = ALIGN (32); diff --git a/src/platform/apollolake/include/platform/lib/memory.h b/src/platform/apollolake/include/platform/lib/memory.h index 357c4bae9cff..20407b54cc95 100644 --- a/src/platform/apollolake/include/platform/lib/memory.h +++ b/src/platform/apollolake/include/platform/lib/memory.h @@ -121,8 +121,6 @@ * +--------------------------------------------------------------------------+ * | Offset | Region | Size | * +------------------+-------------------------+-----------------------------+ - * | HP_SRAM_BASE | DMA | HEAP_HP_BUFFER_SIZE | - * +------------------+-------------------------+-----------------------------+ * | SRAM_SW_REG_BASE | SW Registers W0 | SRAM_SW_REG_SIZE | * +------------------+-------------------------+-----------------------------+ * | SRAM_OUTBOX_BASE | Outbox W0 | SRAM_MAILBOX_SIZE | @@ -164,14 +162,6 @@ #define HP_SRAM_BASE 0xBE000000 #define HP_SRAM_MASK 0xFF000000 -/* HP SRAM Heap */ -#define HEAP_HP_BUFFER_BASE HP_SRAM_BASE -#define HEAP_HP_BUFFER_SIZE 0xA000 - -#define HEAP_HP_BUFFER_BLOCK_SIZE 0x180 -#define HEAP_HP_BUFFER_COUNT \ - (HEAP_HP_BUFFER_SIZE / HEAP_HP_BUFFER_BLOCK_SIZE) - /* HP SRAM windows */ /* window 3 */ @@ -226,7 +216,7 @@ #define SRAM_ALIAS_OFFSET 0x20000000 -#define SRAM_WND_BASE (HEAP_HP_BUFFER_BASE + HEAP_HP_BUFFER_SIZE) +#define SRAM_WND_BASE (HP_SRAM_BASE) #define HP_SRAM_VECBASE_RESET (HP_SRAM_WIN0_BASE + HP_SRAM_WIN0_SIZE) #define HP_SRAM_VECBASE_OFFSET 0x0 @@ -282,7 +272,8 @@ HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512 + \ HEAP_RT_COUNT1024 * 1024) -#define HEAP_BUFFER_SIZE 0x2000 +#define HEAP_BUFFER_SIZE 0x10000 + #define HEAP_BUFFER_BLOCK_SIZE 0x180 #define HEAP_BUFFER_COUNT (HEAP_BUFFER_SIZE / HEAP_BUFFER_BLOCK_SIZE) @@ -377,7 +368,7 @@ #define PLATFORM_HEAP_SYSTEM PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_SYSTEM_RUNTIME PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_RUNTIME 1 -#define PLATFORM_HEAP_BUFFER 3 +#define PLATFORM_HEAP_BUFFER 2 /* Stack configuration */ #define SOF_LP_STACK_SIZE 0x1000 @@ -428,7 +419,8 @@ #define IMR_BOOT_LDR_BSS_SIZE 0x10000 /* Temporary stack place for boot_ldr */ -#define BOOT_LDR_STACK_BASE HEAP_HP_BUFFER_BASE +#define BOOT_LDR_STACK_BASE (HP_SRAM_BASE + HP_SRAM_SIZE - \ + SOF_STACK_TOTAL_SIZE) #define BOOT_LDR_STACK_SIZE SOF_STACK_TOTAL_SIZE /** \brief Manifest base address in IMR - used by boot loader copy procedure. */ diff --git a/src/platform/baytrail/include/platform/lib/memory.h b/src/platform/baytrail/include/platform/lib/memory.h index 0492ed4ece5d..63a9de9a0bb3 100644 --- a/src/platform/baytrail/include/platform/lib/memory.h +++ b/src/platform/baytrail/include/platform/lib/memory.h @@ -16,6 +16,8 @@ void platform_init_memmap(void); #endif +#define PLATFORM_DCACHE_ALIGN sizeof(void *) + /* physical DSP addresses */ #define SHIM_BASE 0xFF340000 @@ -86,13 +88,13 @@ void platform_init_memmap(void); /* Heap section sizes for module pool */ #define HEAP_RT_COUNT8 0 -#define HEAP_RT_COUNT16 48 -#define HEAP_RT_COUNT32 48 +#define HEAP_RT_COUNT16 32 +#define HEAP_RT_COUNT32 32 #define HEAP_RT_COUNT64 32 #define HEAP_RT_COUNT128 32 #define HEAP_RT_COUNT256 32 -#define HEAP_RT_COUNT512 4 -#define HEAP_RT_COUNT1024 4 +#define HEAP_RT_COUNT512 2 +#define HEAP_RT_COUNT1024 1 /* Heap section sizes for system runtime heap */ #define HEAP_SYS_RT_COUNT64 64 diff --git a/src/platform/cannonlake/cannonlake.x.in b/src/platform/cannonlake/cannonlake.x.in index e5696cc1bcd9..3837e9975657 100644 --- a/src/platform/cannonlake/cannonlake.x.in +++ b/src/platform/cannonlake/cannonlake.x.in @@ -81,9 +81,6 @@ MEMORY sof_fw : org = SOF_FW_BASE, len = SOF_FW_MAX_SIZE - buffer_hp_heap : - org = HEAP_HP_BUFFER_BASE, - len = HEAP_HP_BUFFER_SIZE wnd0 : org = HP_SRAM_WIN0_BASE, len = HP_SRAM_WIN0_SIZE @@ -125,7 +122,6 @@ PHDRS vector_double_lit_phdr PT_LOAD; vector_double_text_phdr PT_LOAD; sof_fw_phdr PT_LOAD; - buffer_hp_heap_phdr PT_LOAD; wnd0_phdr PT_LOAD; wnd1_phdr PT_LOAD; wnd2_phdr PT_LOAD; @@ -153,14 +149,6 @@ SECTIONS _MemoryExceptionVector_text_end = ABSOLUTE(.); } >vector_memory_text :vector_memory_text_phdr - .buffer_hp_heap (NOLOAD) : ALIGN(8) - { - . = ALIGN (32); - _buffer_hp_heap_start = ABSOLUTE(.); - . = . + HEAP_HP_BUFFER_SIZE; - _buffer_hp_heap_end = ABSOLUTE(.); - } >buffer_hp_heap :buffer_hp_heap_phdr - .wnd0 (NOLOAD) : ALIGN(8) { . = ALIGN (32); diff --git a/src/platform/cannonlake/include/platform/lib/memory.h b/src/platform/cannonlake/include/platform/lib/memory.h index 179e547375b1..e4a83886e2ea 100644 --- a/src/platform/cannonlake/include/platform/lib/memory.h +++ b/src/platform/cannonlake/include/platform/lib/memory.h @@ -143,8 +143,6 @@ * +------------------+-------------------------+-----------------------------+ * | SRAM_TRACE_BASE | Trace Buffer W3 | SRAM_TRACE_SIZE | * +------------------+-------------------------+-----------------------------+ - * | HP_SRAM_BASE | DMA | HEAP_HP_BUFFER_SIZE | - * +------------------+-------------------------+-----------------------------+ * | SOF_FW_START | text | | * | | data | | * | | ----------------------- | | @@ -220,18 +218,8 @@ #define HP_SRAM_WIN3_BASE SRAM_TRACE_BASE #define HP_SRAM_WIN3_SIZE SRAM_TRACE_SIZE -#define HEAP_HP_BUFFER_BASE (SRAM_TRACE_BASE + SRAM_TRACE_SIZE) -#if CONFIG_COMP_KPB -#define HEAP_HP_BUFFER_SIZE 0x40000 -#else -#define HEAP_HP_BUFFER_SIZE 0x20000 -#endif -#define HEAP_HP_BUFFER_BLOCK_SIZE 0x100 -#define HEAP_HP_BUFFER_COUNT \ - (HEAP_HP_BUFFER_SIZE / HEAP_HP_BUFFER_BLOCK_SIZE) - /* HP SRAM Base */ -#define HP_SRAM_VECBASE_RESET (HEAP_HP_BUFFER_BASE + HEAP_HP_BUFFER_SIZE) +#define HP_SRAM_VECBASE_RESET (SRAM_TRACE_BASE + SRAM_TRACE_SIZE) /* text and data share the same HP L2 SRAM on Cannonlake */ #define SOF_FW_START (HP_SRAM_VECBASE_RESET + 0x400) @@ -266,7 +254,7 @@ HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512 + \ HEAP_RT_COUNT1024 * 1024) -#define HEAP_BUFFER_SIZE 0xF000 +#define HEAP_BUFFER_SIZE 0x50000 #define HEAP_BUFFER_BLOCK_SIZE 0x180 #define HEAP_BUFFER_COUNT (HEAP_BUFFER_SIZE / HEAP_BUFFER_BLOCK_SIZE) @@ -374,7 +362,7 @@ #define PLATFORM_HEAP_SYSTEM PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_SYSTEM_RUNTIME PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_RUNTIME 1 -#define PLATFORM_HEAP_BUFFER 3 +#define PLATFORM_HEAP_BUFFER 2 /* Stack configuration */ #define SOF_LP_STACK_SIZE 0x1000 @@ -420,7 +408,8 @@ #define IMR_BOOT_LDR_BSS_SIZE 0x10000 /* Temporary stack place for boot_ldr */ -#define BOOT_LDR_STACK_BASE HEAP_HP_BUFFER_BASE +#define BOOT_LDR_STACK_BASE (HP_SRAM_BASE + HP_SRAM_SIZE - \ + SOF_STACK_TOTAL_SIZE) #define BOOT_LDR_STACK_SIZE SOF_STACK_TOTAL_SIZE #define uncache_to_cache(address) \ diff --git a/src/platform/haswell/include/platform/lib/memory.h b/src/platform/haswell/include/platform/lib/memory.h index 9b1c36f1b0b4..eb035e12bf59 100644 --- a/src/platform/haswell/include/platform/lib/memory.h +++ b/src/platform/haswell/include/platform/lib/memory.h @@ -16,8 +16,10 @@ void platform_init_memmap(void); #endif -/* physical DSP addresses */ +/* data cache line alignment */ +#define PLATFORM_DCACHE_ALIGN sizeof(void *) +/* physical DSP addresses */ #define SHIM_SIZE 0x00001000 @@ -97,7 +99,7 @@ void platform_init_memmap(void); #define HEAP_SYS_RT_COUNT1024 4 /* Heap configuration */ -#define SOF_DATA_SIZE 0xC000 +#define SOF_DATA_SIZE 0xD000 #define HEAP_SYSTEM_BASE (DRAM0_BASE + SOF_DATA_SIZE) #define HEAP_SYSTEM_SIZE 0x4000 diff --git a/src/platform/icelake/icelake.x.in b/src/platform/icelake/icelake.x.in index 0da6ed9ee060..e768bdfd1a85 100644 --- a/src/platform/icelake/icelake.x.in +++ b/src/platform/icelake/icelake.x.in @@ -81,9 +81,6 @@ MEMORY sof_fw : org = SOF_FW_BASE, len = SOF_FW_MAX_SIZE - buffer_hp_heap : - org = HEAP_HP_BUFFER_BASE, - len = HEAP_HP_BUFFER_SIZE wnd0 : org = HP_SRAM_WIN0_BASE, len = HP_SRAM_WIN0_SIZE @@ -128,7 +125,6 @@ PHDRS vector_double_lit_phdr PT_LOAD; vector_double_text_phdr PT_LOAD; sof_fw_phdr PT_LOAD; - buffer_hp_heap_phdr PT_LOAD; wnd0_phdr PT_LOAD; wnd1_phdr PT_LOAD; wnd2_phdr PT_LOAD; @@ -157,14 +153,6 @@ SECTIONS _MemoryExceptionVector_text_end = ABSOLUTE(.); } >vector_memory_text :vector_memory_text_phdr - .buffer_hp_heap (NOLOAD) : ALIGN(8) - { - . = ALIGN (32); - _buffer_hp_heap_start = ABSOLUTE(.); - . = . + HEAP_HP_BUFFER_SIZE; - _buffer_hp_heap_end = ABSOLUTE(.); - } >buffer_hp_heap :buffer_hp_heap_phdr - .wnd0 (NOLOAD) : ALIGN(8) { . = ALIGN (32); diff --git a/src/platform/icelake/include/platform/lib/memory.h b/src/platform/icelake/include/platform/lib/memory.h index 072831b7da1e..047afc00febe 100644 --- a/src/platform/icelake/include/platform/lib/memory.h +++ b/src/platform/icelake/include/platform/lib/memory.h @@ -143,8 +143,6 @@ * +------------------+-------------------------+-----------------------------+ * | SRAM_TRACE_BASE | Trace Buffer W3 | SRAM_TRACE_SIZE | * +------------------+-------------------------+-----------------------------+ - * | HP_SRAM_BASE | DMA | HEAP_HP_BUFFER_SIZE | - * +------------------+-------------------------+-----------------------------+ * | SOF_FW_START | text | | * | | data | | * | | BSS | | @@ -217,14 +215,8 @@ #define HP_SRAM_WIN3_BASE SRAM_TRACE_BASE #define HP_SRAM_WIN3_SIZE SRAM_TRACE_SIZE -#define HEAP_HP_BUFFER_BASE (SRAM_TRACE_BASE + SRAM_TRACE_SIZE) -#define HEAP_HP_BUFFER_SIZE 0x20000 -#define HEAP_HP_BUFFER_BLOCK_SIZE 0x180 -#define HEAP_HP_BUFFER_COUNT \ - (HEAP_HP_BUFFER_SIZE / HEAP_HP_BUFFER_BLOCK_SIZE) - /* HP SRAM Base */ -#define HP_SRAM_VECBASE_RESET (HEAP_HP_BUFFER_BASE + HEAP_HP_BUFFER_SIZE) +#define HP_SRAM_VECBASE_RESET (SRAM_TRACE_BASE + SRAM_TRACE_SIZE) /* text and data share the same HP L2 SRAM on Icelake */ #define SOF_FW_START (HP_SRAM_VECBASE_RESET + 0x400) @@ -259,7 +251,7 @@ HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512 + \ HEAP_RT_COUNT1024 * 1024) -#define HEAP_BUFFER_SIZE 0xF000 +#define HEAP_BUFFER_SIZE 0x20000 #define HEAP_BUFFER_BLOCK_SIZE 0x180 #define HEAP_BUFFER_COUNT (HEAP_BUFFER_SIZE / HEAP_BUFFER_BLOCK_SIZE) @@ -354,7 +346,7 @@ #define PLATFORM_HEAP_SYSTEM PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_SYSTEM_RUNTIME PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_RUNTIME 1 -#define PLATFORM_HEAP_BUFFER 3 +#define PLATFORM_HEAP_BUFFER 2 /* Stack configuration */ #define SOF_LP_STACK_SIZE 0x1000 @@ -400,7 +392,8 @@ #define IMR_BOOT_LDR_BSS_SIZE 0x10000 /* Temporary stack place for boot_ldr */ -#define BOOT_LDR_STACK_BASE HEAP_HP_BUFFER_BASE +#define BOOT_LDR_STACK_BASE (HP_SRAM_BASE + HP_SRAM_SIZE - \ + SOF_STACK_TOTAL_SIZE) #define BOOT_LDR_STACK_SIZE SOF_STACK_TOTAL_SIZE #define uncache_to_cache(address) \ diff --git a/src/platform/imx8/include/platform/lib/memory.h b/src/platform/imx8/include/platform/lib/memory.h index 97c12918aebc..c6b58cf9af80 100644 --- a/src/platform/imx8/include/platform/lib/memory.h +++ b/src/platform/imx8/include/platform/lib/memory.h @@ -13,6 +13,9 @@ #include #include +/* data cache line alignment */ +#define PLATFORM_DCACHE_ALIGN DCACHE_LINE_SIZE + /* physical DSP addresses */ #define IRAM_BASE 0x596f8000 diff --git a/src/platform/intel/cavs/include/cavs/lib/memory.h b/src/platform/intel/cavs/include/cavs/lib/memory.h index d24afc3805ef..97f02732e793 100644 --- a/src/platform/intel/cavs/include/cavs/lib/memory.h +++ b/src/platform/intel/cavs/include/cavs/lib/memory.h @@ -13,6 +13,9 @@ #include #include +/* data cache line alignment */ +#define PLATFORM_DCACHE_ALIGN DCACHE_LINE_SIZE + #define SRAM_BANK_SIZE (64 * 1024) #define EBB_BANKS_IN_SEGMENT 32 diff --git a/src/platform/intel/cavs/lib/memory.c b/src/platform/intel/cavs/lib/memory.c index 23a027642a92..8abcbfc6e2ba 100644 --- a/src/platform/intel/cavs/lib/memory.c +++ b/src/platform/intel/cavs/lib/memory.c @@ -84,8 +84,6 @@ static struct block_map rt_heap_map[] = { /* Heap blocks for buffers */ static struct block_hdr buf_block[HEAP_BUFFER_COUNT] __aligned(PLATFORM_DCACHE_ALIGN); -static struct block_hdr - hp_buf_block[HEAP_HP_BUFFER_COUNT] __aligned(PLATFORM_DCACHE_ALIGN); static struct block_hdr lp_buf_block[HEAP_LP_BUFFER_COUNT] __aligned(PLATFORM_DCACHE_ALIGN); @@ -94,11 +92,6 @@ static struct block_map buf_heap_map[] = { BLOCK_DEF(HEAP_BUFFER_BLOCK_SIZE, HEAP_BUFFER_COUNT, buf_block), }; -static struct block_map hp_buf_heap_map[] = { - BLOCK_DEF(HEAP_HP_BUFFER_BLOCK_SIZE, HEAP_HP_BUFFER_COUNT, - hp_buf_block), -}; - static struct block_map lp_buf_heap_map[] = { BLOCK_DEF(HEAP_LP_BUFFER_BLOCK_SIZE, HEAP_LP_BUFFER_COUNT, lp_buf_block), @@ -165,29 +158,20 @@ void platform_init_memmap(void) memmap.buffer[0].heap = (uintptr_t)&_buffer_heap; memmap.buffer[0].size = HEAP_BUFFER_SIZE; memmap.buffer[0].info.free = HEAP_BUFFER_SIZE; - memmap.buffer[0].caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_EXT | - SOF_MEM_CAPS_CACHE; - - /* heap hp buffer init */ - memmap.buffer[1].blocks = ARRAY_SIZE(hp_buf_heap_map); - memmap.buffer[1].map = hp_buf_heap_map; - memmap.buffer[1].heap = HEAP_HP_BUFFER_BASE; - memmap.buffer[1].size = HEAP_HP_BUFFER_SIZE; - memmap.buffer[1].info.free = HEAP_HP_BUFFER_SIZE; - memmap.buffer[1].caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP | + memmap.buffer[0].caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_HP | SOF_MEM_CAPS_CACHE | SOF_MEM_CAPS_DMA; /* heap lp buffer init */ - memmap.buffer[2].blocks = ARRAY_SIZE(lp_buf_heap_map); - memmap.buffer[2].map = lp_buf_heap_map; - memmap.buffer[2].heap = HEAP_LP_BUFFER_BASE; - memmap.buffer[2].size = HEAP_LP_BUFFER_SIZE; - memmap.buffer[2].info.free = HEAP_LP_BUFFER_SIZE; - memmap.buffer[2].caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_LP | + memmap.buffer[1].blocks = ARRAY_SIZE(lp_buf_heap_map); + memmap.buffer[1].map = lp_buf_heap_map; + memmap.buffer[1].heap = HEAP_LP_BUFFER_BASE; + memmap.buffer[1].size = HEAP_LP_BUFFER_SIZE; + memmap.buffer[1].info.free = HEAP_LP_BUFFER_SIZE; + memmap.buffer[1].caps = SOF_MEM_CAPS_RAM | SOF_MEM_CAPS_LP | SOF_MEM_CAPS_CACHE | SOF_MEM_CAPS_DMA; /* .total init */ memmap.total.free = HEAP_SYSTEM_T_SIZE + HEAP_SYS_RUNTIME_T_SIZE + - HEAP_RUNTIME_SIZE + HEAP_BUFFER_SIZE + HEAP_HP_BUFFER_SIZE + + HEAP_RUNTIME_SIZE + HEAP_BUFFER_SIZE + HEAP_LP_BUFFER_SIZE; } diff --git a/src/platform/intel/cavs/platform.c b/src/platform/intel/cavs/platform.c index 3af1f42ba767..2c750f0dc8c1 100644 --- a/src/platform/intel/cavs/platform.c +++ b/src/platform/intel/cavs/platform.c @@ -406,13 +406,12 @@ int platform_init(struct sof *sof) CONFIG_SYSTICK_PERIOD); scheduler_init_ll(platform_timer_domain); - /* init low latency multi channel DW-DMA domain and scheduler */ + /* init low latency single channel DW-DMA domain and scheduler */ platform_dma_domain = - dma_multi_chan_domain_init( - &dma[PLATFORM_DW_DMA_INDEX], - PLATFORM_NUM_DW_DMACS, - PLATFORM_DEFAULT_CLOCK, - IS_ENABLED(CONFIG_DW_DMA_AGGREGATED_IRQ)); + dma_single_chan_domain_init + (&dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK); scheduler_init_ll(platform_dma_domain); /* FIXME: do not init the system agent until long running tasks are diff --git a/src/platform/suecreek/include/platform/lib/memory.h b/src/platform/suecreek/include/platform/lib/memory.h index 88d605984122..523d7c79ca20 100644 --- a/src/platform/suecreek/include/platform/lib/memory.h +++ b/src/platform/suecreek/include/platform/lib/memory.h @@ -249,7 +249,7 @@ HEAP_RT_COUNT256 * 256 + HEAP_RT_COUNT512 * 512 + \ HEAP_RT_COUNT1024 * 1024) -#define HEAP_BUFFER_SIZE 0xF000 +#define HEAP_BUFFER_SIZE 0x10000 #define HEAP_BUFFER_BLOCK_SIZE 0x180 #define HEAP_BUFFER_COUNT (HEAP_BUFFER_SIZE / HEAP_BUFFER_BLOCK_SIZE) @@ -280,12 +280,6 @@ SRAM_BANK_SIZE) #define SOF_CORE_S_T_SIZE ((PLATFORM_CORE_COUNT - 1) * SOF_CORE_S_SIZE) -#define HEAP_HP_BUFFER_COUNT 0 -#define HEAP_HP_BUFFER_BLOCK_SIZE 0 -#define HEAP_HP_BUFFER_BASE 0 -#define HEAP_HP_BUFFER_SIZE 0 - - /* * The LP SRAM Heap and Stack on Suecreek are organised like this :- * @@ -353,7 +347,7 @@ #define PLATFORM_HEAP_SYSTEM PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_SYSTEM_RUNTIME PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_RUNTIME 1 -#define PLATFORM_HEAP_BUFFER 3 +#define PLATFORM_HEAP_BUFFER 2 /* Stack configuration */ #define SOF_LP_STACK_SIZE 0x1000 @@ -402,7 +396,8 @@ #define BOOT_LDR_BSS_SIZE 0x100 /* Temporary stack place for boot_ldr */ -#define BOOT_LDR_STACK_BASE HEAP_HP_BUFFER_BASE +#define BOOT_LDR_STACK_BASE (HP_SRAM_BASE + HP_SRAM_SIZE - \ + SOF_STACK_TOTAL_SIZE) #define BOOT_LDR_STACK_SIZE SOF_STACK_TOTAL_SIZE /* code loader entry point for base fw */ diff --git a/src/platform/suecreek/suecreek.x.in b/src/platform/suecreek/suecreek.x.in index c51d494c340d..7e297087561d 100644 --- a/src/platform/suecreek/suecreek.x.in +++ b/src/platform/suecreek/suecreek.x.in @@ -87,9 +87,6 @@ MEMORY sof_fw : org = SOF_FW_BASE, len = SOF_FW_MAX_SIZE, - buffer_hp_heap : - org = HEAP_HP_BUFFER_BASE, - len = HEAP_HP_BUFFER_SIZE static_log_entries_seg (!ari) : org = LOG_ENTRY_ELF_BASE, len = LOG_ENTRY_ELF_SIZE @@ -121,7 +118,6 @@ PHDRS vector_double_lit_phdr PT_LOAD; vector_double_text_phdr PT_LOAD; sof_fw_phdr PT_LOAD; - buffer_hp_heap_phdr PT_LOAD; static_log_entries_phdr PT_NOTE; } diff --git a/src/platform/tigerlake/include/platform/lib/memory.h b/src/platform/tigerlake/include/platform/lib/memory.h index 75fb4f42d3a5..5803be7c0277 100644 --- a/src/platform/tigerlake/include/platform/lib/memory.h +++ b/src/platform/tigerlake/include/platform/lib/memory.h @@ -357,7 +357,7 @@ #define PLATFORM_HEAP_SYSTEM PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_SYSTEM_RUNTIME PLATFORM_CORE_COUNT /* one per core */ #define PLATFORM_HEAP_RUNTIME 1 -#define PLATFORM_HEAP_BUFFER 3 +#define PLATFORM_HEAP_BUFFER 2 /* Stack configuration */ #define SOF_LP_STACK_SIZE 0x1000 diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index 4ffb0ef2ac56..5726d6586831 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -2,6 +2,7 @@ add_local_sources(sof dma_multi_chan_domain.c + dma_single_chan_domain.c edf_schedule.c ll_schedule.c schedule.c diff --git a/src/schedule/dma_single_chan_domain.c b/src/schedule/dma_single_chan_domain.c new file mode 100644 index 000000000000..eae298f5ee3e --- /dev/null +++ b/src/schedule/dma_single_chan_domain.c @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DMA_DOMAIN_OWNER_INVALID 0xFFFFFFFF + +struct dma_domain_data { + struct notifier notifier; + int irq; + struct dma_chan_data *channel; + void (*handler)(void *arg); + void *arg; +}; + +struct dma_domain { + struct dma *dma_array; /* pointer to scheduling DMAs */ + uint32_t num_dma; /* number of scheduling DMAs */ + uint32_t owner; /* core owning the scheduling channel */ + bool channel_changed; /* true if we needed to re-register */ + + /* data per core */ + struct dma_domain_data data[PLATFORM_CORE_COUNT]; +}; + +struct ll_schedule_domain_ops dma_single_chan_domain_ops; + +static void dma_single_chan_domain_enable(struct ll_schedule_domain *domain, + int core); + +/** + * \brief Retrieves DMA channel with lowest period. + * \param[in,out] dma_domain Pointer to DMA domain. + * \return Pointer to DMA channel. + */ +static struct dma_chan_data *dma_chan_min_period(struct dma_domain *dma_domain) +{ + struct dma *dmas = dma_domain->dma_array; + struct dma_chan_data *channel = NULL; + int i; + int j; + + /* get currently registered channel if exists */ + if (dma_domain->owner != DMA_DOMAIN_OWNER_INVALID && + dma_domain->data[dma_domain->owner].channel) + channel = dma_domain->data[dma_domain->owner].channel; + + for (i = 0; i < dma_domain->num_dma; ++i) { + /* DMA not probed */ + if (!dmas[i].sref) + continue; + + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + /* channel not set as scheduling source */ + if (!dma_is_scheduling_source(&dmas[i].chan[j])) + continue; + + /* channel not running */ + if (dmas[i].chan[j].status != COMP_STATE_ACTIVE) + continue; + + /* bigger period */ + if (channel && + channel->period <= dmas[i].chan[j].period) + continue; + + channel = &dmas[i].chan[j]; + } + } + + return channel; +} + +/** + * \brief Sends notification event about scheduling DMA channel change. + * \param[in,out] channel Pointer to new scheduling DMA channel. + */ +static void dma_domain_notify_change(struct dma_chan_data *channel) +{ + struct notify_data notify_data; + + notify_data.id = NOTIFIER_ID_DMA_DOMAIN_CHANGE; + notify_data.target_core_mask = + NOTIFIER_TARGET_CORE_ALL_MASK & ~BIT(cpu_get_id()); + notify_data.data_size = sizeof(*channel); + notify_data.data = channel; + + notifier_event(¬ify_data); +} + +/** + * \brief Registers and enables selected DMA interrupt. + * \param[in,out] channel Pointer to DMA channel. + * \param[in,out] data Pointer to DMA domain data. + * \param[in,out] handler Pointer to DMA interrupt handler. + * \param[in,out] arg Pointer to DMA interrupt handler's argument. + * \return Error code. + */ +static int dma_single_chan_domain_irq_register(struct dma_chan_data *channel, + struct dma_domain_data *data, + void (*handler)(void *arg), + void *arg) +{ + int irq = dma_chan_irq(channel->dma, channel->index); + int ret; + + data->irq = interrupt_get_irq(irq, dma_irq_name(channel->dma)); + if (data->irq < 0) + return data->irq; + + ret = interrupt_register(data->irq, handler, arg); + if (ret < 0) + return ret; + + interrupt_enable(data->irq, arg); + + interrupt_mask(data->irq, cpu_get_id()); + + data->channel = channel; + data->handler = handler; + data->arg = arg; + + return ret; +} + +/** + * \brief Unregisters and disables selected DMA interrupt. + * \param[in,out] data Pointer to DMA domain data. + */ +static void dma_single_chan_domain_irq_unregister(struct dma_domain_data *data) +{ + interrupt_disable(data->irq, data->arg); + + interrupt_unregister(data->irq, data->arg); +} + +/** + * \brief Registers task to DMA domain. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] period Period of the scheduled task. + * \param[in,out] task Task to be registered. + * \param[in,out] handler Pointer to DMA interrupt handler. + * \param[in,out] arg Pointer to DMA interrupt handler's argument. + * \return Error code. + * + * Every core registers for the same DMA channel, but only the one on which + * this channel is actually running is the owner. If there is already channel + * running we still need to verify, if the recently started channel doesn't + * have lower period. In such case change of channel is needed and all other + * cores need to be notified. + */ +static int dma_single_chan_domain_register(struct ll_schedule_domain *domain, + uint64_t period, struct task *task, + void (*handler)(void *arg), + void *arg) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct dma_domain_data *data = &dma_domain->data[core]; + struct dma_chan_data *channel; + bool register_needed = true; + int ret; + + /* get running channel with min period */ + channel = dma_chan_min_period(dma_domain); + if (!channel) + return -EINVAL; + + if (data->channel) { + /* channel with min period already registered */ + if (data->channel->period == channel->period) + return 0; + + /* unregister from current channel */ + dma_single_chan_domain_irq_unregister(data); + dma_interrupt(data->channel, DMA_IRQ_MASK); + dma_interrupt(data->channel, DMA_IRQ_CLEAR); + + dma_domain->channel_changed = true; + + /* already registered */ + register_needed = false; + } + + /* register for interrupt */ + ret = dma_single_chan_domain_irq_register(channel, data, handler, arg); + if (ret < 0) + return ret; + + /* enable channel interrupt */ + dma_interrupt(data->channel, DMA_IRQ_UNMASK); + + /* unmask if we are the owner */ + if (dma_domain->owner == core) + interrupt_unmask(data->irq, core); + + /* notify scheduling channel change */ + if (dma_domain->owner != channel->core) + dma_domain_notify_change(channel); + + /* register for source change notifications */ + if (register_needed) + notifier_register(&data->notifier); + + dma_domain->owner = channel->core; + + return ret; +} + +/** + * \brief Checks if any DMA channel is running on current core. + * \param[in,out] dmas Array of possible DMAs. + * \param[in] num Number of DMAs in the array. + * \return True if any DMA channel is running, false otherwise. + */ +static bool dma_chan_is_any_running(struct dma *dmas, uint32_t num) +{ + int core = cpu_get_id(); + int i; + int j; + + for (i = 0; i < num; ++i) { + /* DMA not probed */ + if (!dmas[i].sref) + continue; + + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + /* channel not set as scheduling source */ + if (!dma_is_scheduling_source(&dmas[i].chan[j])) + continue; + + /* channel owned by different core */ + if (core != dmas[i].chan[j].core) + continue; + + if (dmas[i].chan[j].status == COMP_STATE_ACTIVE) + return true; + } + } + + return false; +} + +/** + * \brief Unregisters task from DMA domain if the current core is the owner. + * \param[in,out] domain Pointer to schedule domain. + * \param[in,out] data Pointer to DMA domain data. + * + * If the owner of scheduling DMA channel unregisters, it has to notify + * other cores about the change. + */ +static void dma_domain_unregister_owner(struct ll_schedule_domain *domain, + struct dma_domain_data *data) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + struct dma_chan_data *channel; + int core = cpu_get_id(); + + /* transfers still scheduled on this channel */ + if (data->channel->status == COMP_STATE_ACTIVE) + return; + + dma_single_chan_domain_irq_unregister(data); + dma_interrupt(data->channel, DMA_IRQ_MASK); + dma_interrupt(data->channel, DMA_IRQ_CLEAR); + data->channel = NULL; + + channel = dma_chan_min_period(dma_domain); + if (!channel) { + dma_domain->owner = DMA_DOMAIN_OWNER_INVALID; + notifier_unregister(&data->notifier); + + return; + } + + /* change owner */ + dma_domain->owner = channel->core; + + /* notify scheduling channel change */ + dma_domain_notify_change(channel); + + /* check if there is another channel running */ + if (dma_chan_is_any_running(dmas, dma_domain->num_dma)) { + /* register again and enable */ + dma_single_chan_domain_irq_register(channel, data, + data->handler, data->arg); + dma_interrupt(data->channel, DMA_IRQ_CLEAR); + dma_single_chan_domain_enable(domain, core); + dma_domain->channel_changed = true; + } +} + +/** + * \brief Unregisters task from DMA domain. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] num_tasks Number of currently scheduled tasks. + */ +static void dma_single_chan_domain_unregister(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + int core = cpu_get_id(); + struct dma_domain_data *data = &dma_domain->data[core]; + + /* channel not registered */ + if (!data->channel) + return; + + /* unregister domain owner */ + if (dma_domain->owner == core) + return dma_domain_unregister_owner(domain, data); + + /* some channel still running, so return */ + if (dma_chan_is_any_running(dmas, dma_domain->num_dma)) + return; + + /* no more transfers scheduled on this core */ + dma_single_chan_domain_irq_unregister(data); + data->channel = NULL; + + notifier_unregister(&data->notifier); +} + +/** + * \brief Unmasks scheduling DMA channel's interrupt. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] core Core on which interrupt should be unmasked. + */ +static void dma_single_chan_domain_enable(struct ll_schedule_domain *domain, + int core) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma_domain_data *data = &dma_domain->data[core]; + + dma_interrupt(data->channel, DMA_IRQ_UNMASK); + + interrupt_unmask(data->irq, core); +} + +/** + * \brief Masks scheduling DMA channel's interrupt. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] core Core on which interrupt should be masked. + */ +static void dma_single_chan_domain_disable(struct ll_schedule_domain *domain, + int core) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma_domain_data *data = &dma_domain->data[core]; + + interrupt_mask(data->irq, core); +} + +/** + * \brief Calculates domain's last tick. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] start Offset of last calculated tick. + */ +static void dma_single_chan_domain_set(struct ll_schedule_domain *domain, + uint64_t start) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma_domain_data *data = &dma_domain->data[cpu_get_id()]; + uint64_t ticks; + + if (dma_domain->channel_changed) { + domain->last_tick = platform_timer_get(platform_timer); + + dma_domain->channel_changed = false; + } else { + ticks = domain->ticks_per_ms * data->channel->period / 1000 + + start; + + domain->last_tick = domain->last_tick ? ticks : start; + } +} + +/** + * \brief Clears scheduling DMA channel's interrupt. + * \param[in,out] domain Pointer to schedule domain. + */ +static void dma_single_chan_domain_clear(struct ll_schedule_domain *domain) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma_domain_data *data = &dma_domain->data[cpu_get_id()]; + + dma_interrupt(data->channel, DMA_IRQ_CLEAR); +} + +/** + * \brief Checks if given task should be executed. + * \param[in,out] domain Pointer to schedule domain. + * \param[in,out] task Task to be checked. + * \return True is task should be executed, false otherwise. + */ +static bool dma_single_chan_domain_is_pending(struct ll_schedule_domain *domain, + struct task *task) +{ + return task->start <= platform_timer_get(platform_timer); +} + +/** + * \brief Scheduling DMA channel change notification handling. + * \param[in] message Id of the notification. + * \param[in,out] data Pointer to notification data. + * \param[in,out] event_data Pointer to notification event data. + */ +static void dma_domain_changed(int message, void *data, void *event_data) +{ + struct ll_schedule_domain *domain = data; + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct dma_domain_data *domain_data = &dma_domain->data[core]; + + /* unregister from current DMA channel */ + dma_single_chan_domain_irq_unregister(domain_data); + + if (domain_data->channel->core == core) { + dma_interrupt(domain_data->channel, DMA_IRQ_MASK); + dma_interrupt(domain_data->channel, DMA_IRQ_CLEAR); + } + + /* register to the new DMA channel */ + if (dma_single_chan_domain_irq_register(event_data, domain_data, + domain_data->handler, + domain_data->arg) < 0) + return; + + dma_single_chan_domain_enable(domain, core); +} + +/** + * \brief Initializes DMA single channel scheduling domain. + * \param[in,out] dma_array Array of DMAs to be scheduled on. + * \param[in] num_dma Number of DMAs passed as dma_array. + * \param[in] clk Platform clock to base calculations on. + * \return Pointer to initialized scheduling domain object. + */ +struct ll_schedule_domain *dma_single_chan_domain_init(struct dma *dma_array, + uint32_t num_dma, + int clk) +{ + struct ll_schedule_domain *domain; + struct dma_domain *dma_domain; + int i; + + domain = domain_init(SOF_SCHEDULE_LL_DMA, clk, + &dma_single_chan_domain_ops); + + dma_domain = rzalloc(RZONE_SYS | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, + sizeof(*dma_domain)); + dma_domain->dma_array = dma_array; + dma_domain->num_dma = num_dma; + dma_domain->owner = DMA_DOMAIN_OWNER_INVALID; + + /* register notifiers */ + for (i = 0; i < PLATFORM_CORE_COUNT; ++i) { + dma_domain->data[i].notifier.cb = dma_domain_changed; + dma_domain->data[i].notifier.cb_data = domain; + dma_domain->data[i].notifier.id = NOTIFIER_ID_DMA_DOMAIN_CHANGE; + } + + ll_sch_domain_set_pdata(domain, dma_domain); + + return domain; +} + +struct ll_schedule_domain_ops dma_single_chan_domain_ops = { + .domain_register = dma_single_chan_domain_register, + .domain_unregister = dma_single_chan_domain_unregister, + .domain_enable = dma_single_chan_domain_enable, + .domain_disable = dma_single_chan_domain_disable, + .domain_set = dma_single_chan_domain_set, + .domain_clear = dma_single_chan_domain_clear, + .domain_is_pending = dma_single_chan_domain_is_pending, +}; diff --git a/src/schedule/ll_schedule.c b/src/schedule/ll_schedule.c index 20530606d03c..3938beda2440 100644 --- a/src/schedule/ll_schedule.c +++ b/src/schedule/ll_schedule.c @@ -115,9 +115,6 @@ static void schedule_ll_clients_enable(struct ll_schedule_data *sch) static void schedule_ll_clients_reschedule(struct ll_schedule_data *sch) { - /* clear domain */ - domain_clear(sch->domain); - /* rearm only if there is work to do */ if (atomic_read(&sch->domain->total_num_tasks)) { domain_set(sch->domain, sch->domain->last_tick); @@ -128,12 +125,22 @@ static void schedule_ll_clients_reschedule(struct ll_schedule_data *sch) static void schedule_ll_tasks_run(void *data) { struct ll_schedule_data *sch = data; + uint32_t num_clients; uint32_t flags; domain_disable(sch->domain, cpu_get_id()); irq_local_disable(flags); + spin_lock(sch->domain->lock); + + /* clear domain only if all clients are done */ + num_clients = atomic_sub(&sch->domain->num_clients, 1); + if (!num_clients) + domain_clear(sch->domain); + + spin_unlock(sch->domain->lock); + /* run tasks if there are any pending */ if (schedule_ll_is_pending(sch)) schedule_ll_tasks_execute(sch); @@ -141,7 +148,7 @@ static void schedule_ll_tasks_run(void *data) spin_lock(sch->domain->lock); /* reschedule only if all clients are done */ - if (!atomic_sub(&sch->domain->num_clients, 1)) + if (!num_clients) schedule_ll_clients_reschedule(sch); spin_unlock(sch->domain->lock); @@ -192,8 +199,10 @@ static void schedule_ll_domain_clear(struct ll_schedule_data *sch) sch->domain->registered[cpu_get_id()] = false; /* reschedule if there is no more clients waiting */ - if (!atomic_sub(&sch->domain->num_clients, 1)) + if (!atomic_sub(&sch->domain->num_clients, 1)) { + domain_clear(sch->domain); schedule_ll_clients_reschedule(sch); + } } spin_unlock(sch->domain->lock); diff --git a/test/cmocka/src/audio/buffer/mock.c b/test/cmocka/src/audio/buffer/mock.c index 2352d0113046..050886b1e7fa 100644 --- a/test/cmocka/src/audio/buffer/mock.c +++ b/test/cmocka/src/audio/buffer/mock.c @@ -17,7 +17,7 @@ TRACE_IMPL() #if !CONFIG_LIBRARY -void *rzalloc(int zone, uint32_t caps, size_t bytes) +void *_zalloc(int zone, uint32_t caps, size_t bytes) { (void)zone; (void)caps; @@ -25,7 +25,8 @@ void *rzalloc(int zone, uint32_t caps, size_t bytes) return malloc(bytes); } -void *rballoc(int zone, uint32_t caps, size_t bytes) +void *_balloc(int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { (void)zone; (void)caps; @@ -38,7 +39,8 @@ void rfree(void *ptr) free(ptr); } -void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes) +void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { (void)zone; (void)caps; diff --git a/test/cmocka/src/audio/component/mock.c b/test/cmocka/src/audio/component/mock.c index d75efde20eb0..1a9c5deeca0f 100644 --- a/test/cmocka/src/audio/component/mock.c +++ b/test/cmocka/src/audio/component/mock.c @@ -17,7 +17,7 @@ TRACE_IMPL() #if !CONFIG_LIBRARY -void *rzalloc(int zone, uint32_t caps, size_t bytes) +void *_zalloc(int zone, uint32_t caps, size_t bytes) { (void)zone; (void)caps; diff --git a/test/cmocka/src/audio/kpb/kpb_mock.c b/test/cmocka/src/audio/kpb/kpb_mock.c index 9590d8e0ad1e..6951625436ba 100644 --- a/test/cmocka/src/audio/kpb/kpb_mock.c +++ b/test/cmocka/src/audio/kpb/kpb_mock.c @@ -21,7 +21,8 @@ TRACE_IMPL() -void *rballoc(int zone, uint32_t caps, size_t bytes) +void *_balloc(int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { (void)zone; (void)caps; @@ -29,7 +30,7 @@ void *rballoc(int zone, uint32_t caps, size_t bytes) return malloc(bytes); } -void *rzalloc(int zone, uint32_t caps, size_t bytes) +void *_zalloc(int zone, uint32_t caps, size_t bytes) { (void)zone; (void)caps; @@ -42,7 +43,8 @@ void rfree(void *ptr) free(ptr); } -void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes) +void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { (void)zone; (void)caps; diff --git a/test/cmocka/src/audio/mixer/mock.c b/test/cmocka/src/audio/mixer/mock.c index c470828c2a69..72e81cdd9ce5 100644 --- a/test/cmocka/src/audio/mixer/mock.c +++ b/test/cmocka/src/audio/mixer/mock.c @@ -20,7 +20,8 @@ TRACE_IMPL() -void *rballoc(int zone, uint32_t caps, size_t bytes) +void *_balloc(int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { (void)zone; (void)caps; @@ -28,7 +29,7 @@ void *rballoc(int zone, uint32_t caps, size_t bytes) return malloc(bytes); } -void *rzalloc(int zone, uint32_t caps, size_t bytes) +void *_zalloc(int zone, uint32_t caps, size_t bytes) { (void)zone; (void)caps; @@ -41,7 +42,8 @@ void rfree(void *ptr) free(ptr); } -void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes) +void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { (void)zone; (void)caps; @@ -53,27 +55,6 @@ void pipeline_xrun(struct pipeline *p, struct comp_dev *dev, int32_t bytes) { } -int comp_set_sink_buffer(struct comp_dev *dev, uint32_t period_bytes, - uint32_t periods) -{ - struct comp_buffer *sinkb; - int ret = 0; - - sinkb = list_first_item(&dev->bsink_list, struct comp_buffer, - source_list); - - if (!sinkb->sink->is_dma_connected) { - ret = buffer_set_size(sinkb, period_bytes * periods); - if (ret < 0) - return ret; - } else if (sinkb->size < period_bytes * periods) { - ret = -EINVAL; - return ret; - } - - return ret; -} - int comp_set_state(struct comp_dev *dev, int cmd) { return 0; diff --git a/test/cmocka/src/audio/mux/util.h b/test/cmocka/src/audio/mux/util.h index b77153ced9f4..224d6e4e31aa 100644 --- a/test/cmocka/src/audio/mux/util.h +++ b/test/cmocka/src/audio/mux/util.h @@ -29,7 +29,7 @@ static inline struct comp_buffer *create_test_sink(struct comp_dev *dev, buffer->sink->params.channels = channels; buffer->free = 0; buffer->avail = 0; - buffer->ipc_buffer.comp.pipeline_id = pipeline_id; + buffer->pipeline_id = pipeline_id; return buffer; } @@ -57,7 +57,7 @@ static inline struct comp_buffer *create_test_source(struct comp_dev *dev, buffer->source->params.channels = channels; buffer->free = 0; buffer->avail = 0; - buffer->ipc_buffer.comp.pipeline_id = pipeline_id; + buffer->pipeline_id = pipeline_id; return buffer; } diff --git a/test/cmocka/src/audio/pipeline/pipeline_mocks.c b/test/cmocka/src/audio/pipeline/pipeline_mocks.c index 04633c00506e..50d19e2cd0c1 100644 --- a/test/cmocka/src/audio/pipeline/pipeline_mocks.c +++ b/test/cmocka/src/audio/pipeline/pipeline_mocks.c @@ -65,7 +65,7 @@ void cpu_power_down_core(void) { } void notifier_notify(void) { } -struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id) +struct ipc_comp_dev *ipc_get_comp_by_id(struct ipc *ipc, uint32_t id) { (void)ipc; (void)id; @@ -73,6 +73,16 @@ struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id) return NULL; } +struct ipc_comp_dev *ipc_get_comp_by_ppl_id(struct ipc *ipc, uint16_t type, + uint32_t ppl_id) +{ + (void)ipc; + (void)type; + (void)ppl_id; + + return NULL; +} + void heap_trace_all(int force) { (void)force; diff --git a/test/cmocka/src/audio/pipeline/pipeline_mocks.h b/test/cmocka/src/audio/pipeline/pipeline_mocks.h index c22a8723254c..795c69ed00a7 100644 --- a/test/cmocka/src/audio/pipeline/pipeline_mocks.h +++ b/test/cmocka/src/audio/pipeline/pipeline_mocks.h @@ -23,7 +23,10 @@ int arch_cpu_is_core_enabled(int id); void cpu_power_down_core(void); -struct ipc_comp_dev *ipc_get_comp(struct ipc *ipc, uint32_t id); +struct ipc_comp_dev *ipc_get_comp_by_id(struct ipc *ipc, uint32_t id); + +struct ipc_comp_dev *ipc_get_comp_by_ppl_id(struct ipc *ipc, uint16_t type, + uint32_t ppl_id); void notifier_notify(void); diff --git a/tools/testbench/alloc.c b/tools/testbench/alloc.c index 7ce3aeeb8f19..3eb9e90ee07d 100644 --- a/tools/testbench/alloc.c +++ b/tools/testbench/alloc.c @@ -16,12 +16,12 @@ /* testbench mem alloc definition */ -void *rmalloc(int zone, uint32_t caps, size_t bytes) +void *_malloc(int zone, uint32_t caps, size_t bytes) { return malloc(bytes); } -void *rzalloc(int zone, uint32_t caps, size_t bytes) +void *_zalloc(int zone, uint32_t caps, size_t bytes) { return calloc(bytes, 1); } @@ -31,17 +31,19 @@ void rfree(void *ptr) free(ptr); } -void *rballoc(int zone, uint32_t caps, size_t bytes) +void *_balloc(int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { return malloc(bytes); } -void *rrealloc(void *ptr, int zone, uint32_t caps, size_t bytes) +void *_realloc(void *ptr, int zone, uint32_t caps, size_t bytes) { return realloc(ptr, bytes); } -void *rbrealloc(void *ptr, int zone, uint32_t caps, size_t bytes) +void *_brealloc(void *ptr, int zone, uint32_t caps, size_t bytes, + uint32_t alignment) { return realloc(ptr, bytes); } diff --git a/tools/testbench/common_test.c b/tools/testbench/common_test.c index 9c9c5e946446..e820a651aca2 100644 --- a/tools/testbench/common_test.c +++ b/tools/testbench/common_test.c @@ -64,7 +64,7 @@ int tb_pipeline_start(struct ipc *ipc, int nch, } /* Get IPC component device for pipeline */ - pcm_dev = ipc_get_comp(ipc, ipc_pipe->sched_id); + pcm_dev = ipc_get_comp_by_id(ipc, ipc_pipe->sched_id); if (!pcm_dev) { fprintf(stderr, "error: ipc get comp\n"); return -EINVAL; @@ -136,7 +136,7 @@ int tb_pipeline_params(struct ipc *ipc, int nch, } /* get scheduling component device for pipeline*/ - pcm_dev = ipc_get_comp(ipc, ipc_pipe->sched_id); + pcm_dev = ipc_get_comp_by_id(ipc, ipc_pipe->sched_id); if (!pcm_dev) { fprintf(stderr, "error: ipc get comp\n"); return -EINVAL; diff --git a/tools/testbench/testbench.c b/tools/testbench/testbench.c index 0175ae151da9..0f0cdfa153f7 100644 --- a/tools/testbench/testbench.c +++ b/tools/testbench/testbench.c @@ -217,13 +217,13 @@ int main(int argc, char **argv) } /* Get pointers to fileread and filewrite */ - pcm_dev = ipc_get_comp(sof.ipc, fw_id); + pcm_dev = ipc_get_comp_by_id(sof.ipc, fw_id); fwcd = comp_get_drvdata(pcm_dev->cd); - pcm_dev = ipc_get_comp(sof.ipc, fr_id); + pcm_dev = ipc_get_comp_by_id(sof.ipc, fr_id); frcd = comp_get_drvdata(pcm_dev->cd); /* Run pipeline until EOF from fileread */ - pcm_dev = ipc_get_comp(sof.ipc, sched_id); + pcm_dev = ipc_get_comp_by_id(sof.ipc, sched_id); p = pcm_dev->cd->pipeline; ipc_pipe = &p->ipc_pipe;