diff --git a/src/audio/component.c b/src/audio/component.c index 02797941d6b6..e2e670670126 100644 --- a/src/audio/component.c +++ b/src/audio/component.c @@ -640,13 +640,18 @@ void comp_init_performance_data(struct comp_dev *dev) { struct perf_data_item_comp *item = dev->perf_data.perf_data_item; - if (item) + if (item) { perf_data_item_comp_init(item, dev->ipc_config.id, 0); + perf_data_update_module_resources(dev); + } } /* returns true if budget violation occurred */ static bool update_peak_of_measured_cpc(struct comp_dev *dev, size_t measured_cpc) { + if (dev->cpc && measured_cpc > dev->cpc) + perf_data_inc_cpc_violation(dev); + if (measured_cpc <= dev->perf_data.peak_of_measured_cpc) return false; dev->perf_data.peak_of_measured_cpc = measured_cpc; @@ -657,7 +662,8 @@ bool comp_update_performance_data(struct comp_dev *dev, uint32_t cycles_used) { struct perf_data_item_comp *item = dev->perf_data.perf_data_item; - if (perf_meas_get_state() == IPC4_PERF_MEASUREMENTS_STARTED) { + if (perf_meas_get_state() == IPC4_PERF_MEASUREMENTS_STARTED || + IS_ENABLED(CONFIG_SOF_TELEMETRY_MODULE_RESOURCES)) { /* we divide by ibs so we need to check if its set */ if (item && dev->ibs != 0) { item->total_iteration_count++; @@ -720,6 +726,9 @@ void comp_update_ibs_obs_cpc(struct comp_dev *dev) dev->obs = dev_cfg.obs; dev->ibs = dev_cfg.ibs; dev->cpc = dev_cfg.cpc; +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + perf_data_update_module_resources(dev); +#endif #else /* set neutral values */ dev->ll_chunk_size = 0; diff --git a/src/audio/dai-zephyr.c b/src/audio/dai-zephyr.c index c0f04700628b..be20809b7ec5 100644 --- a/src/audio/dai-zephyr.c +++ b/src/audio/dai-zephyr.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -357,6 +358,7 @@ dai_dma_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t bytes, /* fill buffer with silence */ buffer_zero(dd->dma_buffer); + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -502,6 +504,7 @@ dai_dma_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t bytes, /* Increment performance counters */ io_perf_monitor_update_data(dd->io_perf_dai_byte_count, bytes); #endif + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -532,6 +535,7 @@ dai_dma_multi_endpoint_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t fr /* fill buffer with silence */ buffer_zero(dd->dma_buffer); + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -564,6 +568,7 @@ dai_dma_multi_endpoint_cb(struct dai_data *dd, struct comp_dev *dev, uint32_t fr /* update host position (in bytes offset) for drivers */ dd->total_data_processed += bytes; + telemetry_update_dai_hot(dd, dev); return dma_status; } @@ -656,6 +661,7 @@ __cold int dai_common_new(struct dai_data *dd, struct comp_dev *dev, io_perf_monitor_init_data(&dd->io_perf_dai_byte_count, &init_data); } #endif + telemetry_register_dai(dd, dev); return 0; } @@ -790,6 +796,7 @@ __cold void dai_common_free(struct dai_data *dd) #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS io_perf_monitor_release_slot(dd->io_perf_dai_byte_count); #endif + telemetry_unregister_dai(dd); if (dd->group) dai_group_put(dd->group); @@ -1293,6 +1300,7 @@ int dai_common_params(struct dai_data *dd, struct comp_dev *dev, goto out; } + telemetry_register_dai(dd, dev); out: /* * Make sure to free all allocated items, all functions @@ -1559,6 +1567,7 @@ static int dai_comp_trigger_internal(struct dai_data *dd, struct comp_dev *dev, case COMP_TRIGGER_XRUN: comp_info(dev, "XRUN"); dd->xrun = 1; + telemetry_update_dai_hot(dd, dev); COMPILER_FALLTHROUGH; case COMP_TRIGGER_STOP: diff --git a/src/audio/host-zephyr.c b/src/audio/host-zephyr.c index d89364f1f3bf..e74415833346 100644 --- a/src/audio/host-zephyr.c +++ b/src/audio/host-zephyr.c @@ -25,6 +25,7 @@ #include #include #include +#include #include /* zephyr_ll_user_heap() */ #include #include @@ -269,6 +270,7 @@ void host_common_update(struct host_data *hd, struct comp_dev *dev, uint32_t byt #endif hd->total_data_processed += bytes; + telemetry_update_host_hot(hd, dev); /* new local period, update host buffer position blks * local_pos is queried by the ops.position() API @@ -533,6 +535,7 @@ static uint32_t host_get_copy_bytes_normal(struct host_data *hd, struct comp_dev avail_samples, free_samples); } hd->n_skipped++; + telemetry_update_host_hot(hd, dev); } } @@ -842,6 +845,7 @@ __cold void host_common_free(struct host_data *hd) hd->io_perf_host_byte_count = NULL; } #endif + telemetry_unregister_host(hd); /* release DMA channel if not already done by reset */ if (hd->chan_index >= 0) { @@ -1173,6 +1177,7 @@ int host_common_params(struct host_data *hd, struct comp_dev *dev, io_perf_monitor_init_data(&hd->io_perf_host_byte_count, &init_data); } #endif + telemetry_register_host(hd, dev); return 0; diff --git a/src/audio/module_adapter/module/generic.c b/src/audio/module_adapter/module/generic.c index 415b2437c17d..6724e983faa0 100644 --- a/src/audio/module_adapter/module/generic.c +++ b/src/audio/module_adapter/module/generic.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -231,6 +232,7 @@ void *z_impl_mod_balloc_align(struct processing_module *mod, size_t size, size_t res->heap_high_water_mark = res->heap_usage; k_mutex_unlock(&mod_res_lock); + perf_data_update_module_resources(mod->dev); return ptr; } EXPORT_SYMBOL(z_impl_mod_balloc_align); @@ -284,6 +286,7 @@ void *z_impl_mod_alloc_ext(struct processing_module *mod, uint32_t flags, size_t res->heap_high_water_mark = res->heap_usage; k_mutex_unlock(&mod_res_lock); + perf_data_update_module_resources(mod->dev); return ptr; } EXPORT_SYMBOL(z_impl_mod_alloc_ext); @@ -486,6 +489,8 @@ int z_impl_mod_free(struct processing_module *mod, const void *ptr) if (ret < 0) comp_err(mod->dev, "error: could not find memory pointed by %p", ptr); + else + perf_data_update_module_resources(mod->dev); return ret; } @@ -791,6 +796,7 @@ __cold void z_impl_mod_free_all(struct processing_module *mod) /* Make sure resource lists and accounting are reset */ mod_resource_init(mod); + perf_data_update_module_resources(mod->dev); } __cold int module_free(struct processing_module *mod) diff --git a/src/audio/pipeline/pipeline-schedule.c b/src/audio/pipeline/pipeline-schedule.c index 3893cd227c9b..63b9fd979088 100644 --- a/src/audio/pipeline/pipeline-schedule.c +++ b/src/audio/pipeline/pipeline-schedule.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,7 @@ static enum task_state pipeline_task_cmd(struct pipeline *p, } p->trigger.aborted = false; + telemetry_notify_pipeline_change(); return ret; } @@ -144,8 +146,10 @@ static enum task_state pipeline_task_cmd(struct pipeline *p, } } else if (p->trigger.cmd != cmd) { /* PRE stage completed */ - if (p->trigger.delay) + if (p->trigger.delay) { + telemetry_notify_pipeline_change(); return SOF_TASK_STATE_RESCHEDULE; + } /* No delay: the final stage has already run too */ err = SOF_TASK_STATE_RESCHEDULE; } else if (p->status == COMP_STATE_PAUSED) { @@ -163,6 +167,7 @@ static enum task_state pipeline_task_cmd(struct pipeline *p, } p->trigger.cmd = COMP_TRIGGER_NO_ACTION; + telemetry_notify_pipeline_change(); ipc_msg_reply(reply); @@ -240,6 +245,7 @@ static enum task_state ipc3_pipeline_task(void *arg) err = pipeline_xrun_recover(p); if (err < 0) { pipe_err(p, "xrun recovery failed! pipeline is stopped."); + telemetry_notify_pipeline_change(); /* failed - host will stop this pipeline */ return SOF_TASK_STATE_COMPLETED; } @@ -332,6 +338,7 @@ static enum task_state ipc4_pipeline_copy_task(void *arg) err = pipeline_xrun_recover(p); if (err < 0) { pipe_err(p, "xrun recovery failed! pipeline is stopped."); + telemetry_notify_pipeline_change(); /* failed - host will stop this pipeline */ return SOF_TASK_STATE_COMPLETED; } diff --git a/src/audio/pipeline/pipeline-stream.c b/src/audio/pipeline/pipeline-stream.c index 9b015f26cb4d..ac7352947dae 100644 --- a/src/audio/pipeline/pipeline-stream.c +++ b/src/audio/pipeline/pipeline-stream.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -401,7 +402,10 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) if (p->status == COMP_STATE_PAUSED || p->xrun_bytes) { /* The task isn't running, trigger inline */ ret = pipeline_trigger_run(p, host, cmd); - return ret < 0 ? ret : 0; + if (ret < 0) + return ret; + telemetry_notify_pipeline_change(); + return 0; } COMPILER_FALLTHROUGH; @@ -432,9 +436,11 @@ int pipeline_trigger(struct pipeline *p, struct comp_dev *host, int cmd) irq_unlock(flags); #endif /* IPC response will be sent from the task, unless it was paused */ + telemetry_notify_pipeline_change(); return PPL_STATUS_SCHEDULED; } + telemetry_notify_pipeline_change(); return 0; } diff --git a/src/audio/pipeline/pipeline-xrun.c b/src/audio/pipeline/pipeline-xrun.c index fde01cbe699e..09ad9be84457 100644 --- a/src/audio/pipeline/pipeline-xrun.c +++ b/src/audio/pipeline/pipeline-xrun.c @@ -7,6 +7,8 @@ #include #include +#include +#include #include #include #include @@ -163,6 +165,9 @@ void pipeline_xrun(struct pipeline *p, struct comp_dev *dev, if (dev->state != COMP_STATE_ACTIVE) return; + telemetry_notify_pipeline_xrun(p, bytes); + perf_data_inc_xrun(dev); + /* notify all pipeline comps we are in XRUN, and stop copying */ ret = pipeline_trigger(p, p->source_comp, COMP_TRIGGER_XRUN); if (ret < 0) diff --git a/src/debug/telemetry/Kconfig b/src/debug/telemetry/Kconfig index f380a90e1808..7497f2aec270 100644 --- a/src/debug/telemetry/Kconfig +++ b/src/debug/telemetry/Kconfig @@ -27,3 +27,89 @@ config SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS Disabled by default and enabled with IPC. Measurements can be extracted also by IPC. Interfaces measured: IPC, IDC, DMIC, I2S, SNDW, HDA, USB, GPIO, I2c, I3C, UART, SPI, CSI_2, DTF. +if SOF_TELEMETRY + +config SOF_TELEMETRY_EXTENDED_WINDOW + bool "Extended telemetry directory in Memory Window 2 Slot 1" + default y + help + Places a 64-byte-aligned self-describing TEL2 directory header and + cache-line-concentrated hot counter zone after system_tick_info in + the telemetry debug window slot to expose internal SOF resources. + +config SOF_TELEMETRY_HOT_CACHE_STAGING + bool "Stage LL hot-path telemetry updates in L1 D-cache lines" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + default y + help + Accumulates LL hot-path counter increments in 64-byte-aligned L1 D$ + lines and flushes dirty cache lines to the uncached memory window + once at the end of the LL tick. + +config SOF_TELEMETRY_MEMORY + bool "Memory, k_heap, virtual heap, and SRAM page telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + select SYS_HEAP_RUNTIME_STATS + default y + help + Exports live usage, free bytes, high-water marks, and allocation + failure counters for sof_heap, shared_buffer_heap, ll_user_heap, + l3_heap, virtual_buffers_heap (vmh), and fast_get in MW2 Slot 1. + +config SOF_TELEMETRY_VREGION + bool "Virtual page (vpage) and virtual region (vregion) telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW && SOF_VREGIONS + default y + help + Tracks vpage_ctx page usage/fragmentation and per-vregion lifetime + linear allocator and interim k_heap usage in MW2 Slot 1. + +config SOF_TELEMETRY_PIPELINES + bool "Pipeline and LL/DP scheduler telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + default y + help + Exports active pipeline states, periods, priorities, XRUN counts, + and per-core LL/DP task counts and deadline misses in MW2 Slot 1. + +config SOF_TELEMETRY_DAIS + bool "DAI and Host DMA gateway telemetry" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + default y + help + Exports active DAI and Host DMA gateway state, DMA channel, buffer + fill levels, XRUN/skip counts, and total bytes processed in MW2 Slot 1. + +config SOF_TELEMETRY_THREADS + bool "Compact Zephyr thread CPU and stack telemetry in MW2 Slot 1" + depends on SOF_TELEMETRY_EXTENDED_WINDOW + select THREAD_MONITOR + select THREAD_NAME + select THREAD_STACK_INFO + select INIT_STACKS + select THREAD_RUNTIME_STATS + default y + help + Periodically snapshots per-thread CPU load (%) and stack high-water + mark (%) into MW2 Slot 1 without requiring an extra 4 KB debug_stream slot. + +config SOF_TELEMETRY_MODULE_RESOURCES + bool "Per-module heap usage, CPC violation, and I/O telemetry in MW3" + depends on SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + default y + help + Partitions Memory Window 3 to include per-component module_resources + (heap_usage, heap_high_water_mark), peak CPC vs budget, and relocating + io_perf_data_items from .bss into MW3. + +config SOF_TELEMETRY_MAX_COMP_ENTRIES + int "Maximum component entries in Memory Window 3" + depends on SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + range 1 88 if SOF_TELEMETRY_MODULE_RESOURCES + range 1 170 + default 88 if SOF_TELEMETRY_MODULE_RESOURCES + default 170 + help + Maximum number of component performance entries in Memory Window 3. + +endif # SOF_TELEMETRY diff --git a/src/debug/telemetry/performance_monitor.c b/src/debug/telemetry/performance_monitor.c index 49f5427b3982..784dc83709a4 100644 --- a/src/debug/telemetry/performance_monitor.c +++ b/src/debug/telemetry/performance_monitor.c @@ -5,6 +5,7 @@ // Author: Tobiasz Dryjanski #include +#include #include #include #include @@ -28,8 +29,41 @@ LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +#define PERF_DATA_MAX_WINDOW_BYTES SOF_TEL_MW3_MOD_HOT_OFFSET +#else +#define PERF_DATA_MAX_WINDOW_BYTES CONFIG_MEMORY_WIN_3_SIZE +#endif + +#ifdef CONFIG_SOF_TELEMETRY_MAX_COMP_ENTRIES +#define PERFORMANCE_DATA_ENTRIES_COUNT CONFIG_SOF_TELEMETRY_MAX_COMP_ENTRIES +#else #define PERFORMANCE_DATA_ENTRIES_COUNT \ - (CONFIG_MEMORY_WIN_3_SIZE / sizeof(struct perf_data_item_comp)) + (PERF_DATA_MAX_WINDOW_BYTES / sizeof(struct perf_data_item_comp)) +#endif + +BUILD_ASSERT(PERFORMANCE_DATA_ENTRIES_COUNT * sizeof(struct perf_data_item_comp) <= + PERF_DATA_MAX_WINDOW_BYTES, + "PERFORMANCE_DATA_ENTRIES_COUNT exceeds Memory Window 3 perf_data partition"); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +BUILD_ASSERT(SOF_TEL_MW3_MOD_HOT_OFFSET + + PERFORMANCE_DATA_ENTRIES_COUNT * sizeof(struct sof_tel_mod_hot) <= + SOF_TEL_MW3_MOD_COLD_OFFSET, + "mod_hot_data exceeds its Memory Window 3 partition"); +BUILD_ASSERT(SOF_TEL_MW3_MOD_COLD_OFFSET + + PERFORMANCE_DATA_ENTRIES_COUNT * sizeof(struct sof_tel_mod_cold) <= + SOF_TEL_MW3_IO_PERF_OFFSET, + "mod_cold_data exceeds its Memory Window 3 partition"); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS +BUILD_ASSERT(SOF_TEL_MW3_IO_PERF_OFFSET + + SOF_TEL_MW3_MAX_IO_ITEMS * sizeof(struct io_perf_data_item) <= + SOF_TEL_MW3_HDR_OFFSET, + "io_perf_data exceeds its Memory Window 3 partition"); +#endif +BUILD_ASSERT(SOF_TEL_MW3_HDR_OFFSET + sizeof(struct sof_tel_mw3_hdr) <= + CONFIG_MEMORY_WIN_3_SIZE, + "sof_tel_mw3_hdr exceeds Memory Window 3 size"); +#endif SYS_BITARRAY_DEFINE_STATIC(performance_data_bit_array, PERFORMANCE_DATA_ENTRIES_COUNT); @@ -50,6 +84,11 @@ struct perf_bitmap performance_data_bitmap; struct perf_data_item_comp *perf_data; +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +static struct sof_tel_mod_hot *mod_hot_data; +static struct sof_tel_mod_cold *mod_cold_data; +#endif + /* Note that ref. FW used one state per core, all set together to the same state * by one IPC but only for active cores. It may work slightly different in case * where we enable a core while perf meas is started. @@ -157,6 +196,14 @@ int perf_data_free(struct perf_data_item_comp * const item) if (ret < 0) return ret; +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + if (mod_hot_data && idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) { + memset(item, 0, sizeof(*item)); + memset(&mod_hot_data[idx], 0, sizeof(mod_hot_data[idx])); + memset(&mod_cold_data[idx], 0, sizeof(mod_cold_data[idx])); + } +#endif + return 0; } @@ -169,6 +216,13 @@ void perf_data_item_comp_reset(struct perf_data_item_comp *perf) perf->restricted_peak_cycles = 0; perf->item.peak_kcps = 0; perf->item.avg_kcps = 0; + +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + int idx = (int)(perf - perf_data); + + if (mod_hot_data && idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) + memset(&mod_hot_data[idx], 0, sizeof(mod_hot_data[idx])); +#endif } __cold void perf_data_item_comp_init(struct perf_data_item_comp *perf, uint32_t resource_id, @@ -180,8 +234,85 @@ __cold void perf_data_item_comp_init(struct perf_data_item_comp *perf, uint32_t perf->item.resource_id = resource_id; perf->item.is_removed = false; perf->item.power_mode = power_mode; + +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + int idx = (int)(perf - perf_data); + + if (mod_cold_data && idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) { + mod_cold_data[idx].resource_id = resource_id; + mod_cold_data[idx].heap_usage = 0; + mod_cold_data[idx].heap_high_water_mark = 0; + mod_cold_data[idx].configured_cpc = 0; + } +#endif +} + +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +void perf_data_update_module_resources(struct comp_dev *dev) +{ + struct perf_data_item_comp *item; + struct processing_module *mod; + size_t hwm = 0; + size_t usage; + int idx; + + if (!dev || !mod_cold_data || !perf_data) + return; + + item = dev->perf_data.perf_data_item; + if (!item) + return; + + idx = (int)(item - perf_data); + if (idx < 0 || idx >= PERFORMANCE_DATA_ENTRIES_COUNT) + return; + + mod_cold_data[idx].resource_id = dev->ipc_config.id; + mod_cold_data[idx].configured_cpc = dev->cpc; + mod = dev->mod; + if (mod) { + usage = module_adapter_heap_usage(mod, &hwm); + mod_cold_data[idx].heap_usage = (uint32_t)usage; + mod_cold_data[idx].heap_high_water_mark = + MAX(mod_cold_data[idx].heap_high_water_mark, (uint32_t)hwm); + } +} + +void perf_data_inc_cpc_violation(struct comp_dev *dev) +{ + struct perf_data_item_comp *item; + int idx; + + if (!dev || !mod_hot_data || !perf_data) + return; + + item = dev->perf_data.perf_data_item; + if (!item) + return; + + idx = (int)(item - perf_data); + if (idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) + mod_hot_data[idx].cpc_violation_count++; } +void perf_data_inc_xrun(struct comp_dev *dev) +{ + struct perf_data_item_comp *item; + int idx; + + if (!dev || !mod_hot_data || !perf_data) + return; + + item = dev->perf_data.perf_data_item; + if (!item) + return; + + idx = (int)(item - perf_data); + if (idx >= 0 && idx < PERFORMANCE_DATA_ENTRIES_COUNT) + mod_hot_data[idx].xrun_count++; +} +#endif + __cold int free_performance_data(struct perf_data_item_comp *item) { int ret; @@ -432,21 +563,44 @@ __cold int performance_monitor_init(void) perf_bitmap_init(&performance_data_bitmap, &performance_data_bit_array, PERFORMANCE_DATA_ENTRIES_COUNT); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + uint8_t *pmw_base = (uint8_t *)ADSP_PMW; + struct sof_tel_mw3_hdr *hdr = (struct sof_tel_mw3_hdr *)(pmw_base + SOF_TEL_MW3_HDR_OFFSET); + + memset(perf_data, 0, sizeof(*perf_data) * PERFORMANCE_DATA_ENTRIES_COUNT); + mod_hot_data = (struct sof_tel_mod_hot *)(pmw_base + SOF_TEL_MW3_MOD_HOT_OFFSET); + mod_cold_data = (struct sof_tel_mod_cold *)(pmw_base + SOF_TEL_MW3_MOD_COLD_OFFSET); + memset(mod_hot_data, 0, sizeof(*mod_hot_data) * PERFORMANCE_DATA_ENTRIES_COUNT); + memset(mod_cold_data, 0, sizeof(*mod_cold_data) * PERFORMANCE_DATA_ENTRIES_COUNT); + memset(hdr, 0, sizeof(*hdr)); + hdr->magic = SOF_TEL_MW3_MAGIC; + hdr->version = 1; + hdr->max_comp_entries = PERFORMANCE_DATA_ENTRIES_COUNT; + hdr->max_io_entries = SOF_TEL_MW3_MAX_IO_ITEMS; + hdr->mod_hot_offset = SOF_TEL_MW3_MOD_HOT_OFFSET; + hdr->mod_cold_offset = SOF_TEL_MW3_MOD_COLD_OFFSET; + hdr->io_perf_offset = SOF_TEL_MW3_IO_PERF_OFFSET; +#endif + return 0; } -/* init performance monitor using Zephyr */ -SYS_INIT(performance_monitor_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY); +/* init performance monitor before sof_init (POST_KERNEL, 99) creates components */ +SYS_INIT(performance_monitor_init, PRE_KERNEL_2, 0); #ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS -SYS_BITARRAY_DEFINE_STATIC(io_performance_data_bit_array, PERFORMANCE_DATA_ENTRIES_COUNT); - +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES +#define IO_PERFORMANCE_MAX_ENTRIES SOF_TEL_MW3_MAX_IO_ITEMS +#else #define IO_PERFORMANCE_ALLOC_BYTES 0x1000 #define IO_PERFORMANCE_MAX_ENTRIES (IO_PERFORMANCE_ALLOC_BYTES / sizeof(struct io_perf_data_item)) +static struct io_perf_data_item io_perf_data_items[IO_PERFORMANCE_MAX_ENTRIES]; +#endif + +SYS_BITARRAY_DEFINE_STATIC(io_performance_data_bit_array, IO_PERFORMANCE_MAX_ENTRIES); static struct io_perf_monitor_ctx perf_monitor_ctx; -static struct io_perf_data_item io_perf_data_items[IO_PERFORMANCE_MAX_ENTRIES]; __cold int io_perf_monitor_init(void) { @@ -458,8 +612,15 @@ __cold int io_perf_monitor_init(void) k_spinlock_init(&self->lock); k_spinlock_key_t key = k_spin_lock(&self->lock); +#ifdef CONFIG_SOF_TELEMETRY_MODULE_RESOURCES + self->io_perf_data = (struct io_perf_data_item *)((uint8_t *)ADSP_PMW + + SOF_TEL_MW3_IO_PERF_OFFSET); + memset(self->io_perf_data, 0, sizeof(*self->io_perf_data) * IO_PERFORMANCE_MAX_ENTRIES); + self->state = IPC4_PERF_MEASUREMENTS_STARTED; +#else self->io_perf_data = (struct io_perf_data_item *)io_perf_data_items; self->state = IPC4_PERF_MEASUREMENTS_DISABLED; +#endif ret = perf_bitmap_init(&self->io_performance_data_bitmap, &io_performance_data_bit_array, IO_PERFORMANCE_MAX_ENTRIES); diff --git a/src/debug/telemetry/telemetry.c b/src/debug/telemetry/telemetry.c index 676265592a9b..89a3c833e00a 100644 --- a/src/debug/telemetry/telemetry.c +++ b/src/debug/telemetry/telemetry.c @@ -5,12 +5,26 @@ // Author: Tobiasz Dryjanski #include +#include +#include #include #include +#include #include +#include +#include #include +#include +#include +#include #include +#include +#include +#include +#include + +#include "../../audio/copier/host_copier.h" #include #include @@ -23,22 +37,969 @@ #include #include #include +#include LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); -/* Systic variables, one set per core */ +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW +BUILD_ASSERT(sizeof(struct sof_tel_ext_hdr) == 64, "sof_tel_ext_hdr must be 64B"); +BUILD_ASSERT(sizeof(struct sof_tel_hot_line0) == 64, "sof_tel_hot_line0 must be 64B"); +BUILD_ASSERT(sizeof(struct sof_tel_gw_hot) == 8, "sof_tel_gw_hot must be 8B"); +BUILD_ASSERT(sizeof(struct sof_tel_mem_section) == 256, "sof_tel_mem_section must be 256B"); +BUILD_ASSERT(sizeof(struct sof_tel_vregion_section) == 512, "sof_tel_vregion_section must be 512B"); +BUILD_ASSERT(sizeof(struct sof_tel_ppl_section) == 384, "sof_tel_ppl_section must be 384B"); +BUILD_ASSERT(sizeof(struct sof_tel_dai_section) == 256, "sof_tel_dai_section must be 256B"); +BUILD_ASSERT(sizeof(struct sof_tel_thread_section) == 832, "sof_tel_thread_section must be 832B"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, ext_hdr) == 0x0100, "ext_hdr offset != 0x0100"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, hot_line0) == 0x0140, "hot_line0 offset != 0x0140"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, gw_hot) == 0x0180, "gw_hot offset != 0x0180"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, mem_sec) == 0x0240, "mem_sec offset != 0x0240"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, vreg_sec) == 0x0340, "vreg_sec offset != 0x0340"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, ppl_sec) == 0x0540, "ppl_sec offset != 0x0540"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, dai_sec) == 0x06C0, "dai_sec offset != 0x06C0"); +BUILD_ASSERT(offsetof(struct telemetry_wnd_data, thd_sec) == 0x07C0, "thd_sec offset != 0x07C0"); +BUILD_ASSERT(sizeof(struct telemetry_wnd_data) <= 4096, "telemetry_wnd_data exceeds 4KB slot"); +#endif + +/* Systick variables, one set per core */ static int telemetry_systick_counter[CONFIG_MAX_CORE_COUNT]; #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER static struct telemetry_wnd_data *wnd_data; #endif #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS -static int telemetry_prev_ccount[CONFIG_MAX_CORE_COUNT]; static int telemetry_perf_period_sum[CONFIG_MAX_CORE_COUNT]; static int telemetry_perf_period_cnt[CONFIG_MAX_CORE_COUNT]; static struct telemetry_perf_queue telemetry_perf_queue[CONFIG_MAX_CORE_COUNT]; #endif +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW +#define TELEMETRY_BG_REFRESH_PERIOD K_MSEC(1000) + +static struct k_spinlock tel_lock; +static struct k_work_delayable telemetry_bg_dwork; +static bool telemetry_ext_ready; + +struct telemetry_shared_state { +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct sof_tel_hot_line0 hot_line0_cache __aligned(64); + struct sof_tel_gw_hot gw_hot_cache[SOF_TEL_MAX_GATEWAYS] __aligned(64); + bool hot_line0_dirty; + bool gw_line1_dirty; + bool gw_line2_dirty; +#endif + uint32_t total_tick_counter; +} __aligned(64); + +static struct telemetry_shared_state tel_state_storage __aligned(64); + +static inline struct telemetry_shared_state *tel_state(void) +{ + return (struct telemetry_shared_state *)cache_to_uncache( + (__sparse_force void __sparse_cache *)&tel_state_storage); +} + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES +struct tel_ppl_slot { + bool in_use; + uint16_t pipeline_id; + uint16_t xrun_count; + int32_t last_xrun_bytes; +}; + +static struct tel_ppl_slot tel_ppl_slots[SOF_TEL_MAX_PIPELINES]; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_DAIS +struct tel_gw_slot { + bool in_use; + bool is_host; + const void *key; + struct sof_tel_gw_cold cold; +}; + +static struct tel_gw_slot tel_gw_slots[SOF_TEL_MAX_GATEWAYS]; +#endif + +struct telemetry_wnd_data *telemetry_get_wnd_ptr(void) +{ +#ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER + return wnd_data; +#else + return (struct telemetry_wnd_data *)ADSP_DW->slots[SOF_DW_TELEMETRY_SLOT]; +#endif +} + +/* Must be called with tel_lock held */ +static inline struct sof_tel_hot_line0 *telemetry_get_hot_line0_locked(void) +{ +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct telemetry_shared_state *st = tel_state(); + + st->hot_line0_dirty = true; + return &st->hot_line0_cache; +#else + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + return wnd ? &wnd->hot_line0 : NULL; +#endif +} + +/* Must be called with tel_lock held */ +static inline struct sof_tel_gw_hot *telemetry_get_gw_hot_locked(int slot) +{ +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct telemetry_shared_state *st = tel_state(); + + if (slot < 8) + st->gw_line1_dirty = true; + else + st->gw_line2_dirty = true; + return &st->gw_hot_cache[slot]; +#else + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + + return wnd ? &wnd->gw_hot[slot] : NULL; +#endif +} + +/* Must be called with tel_lock held */ +static inline void telemetry_flush_hot_staging_locked(struct telemetry_wnd_data *wnd) +{ +#ifdef CONFIG_SOF_TELEMETRY_HOT_CACHE_STAGING + struct telemetry_shared_state *st = tel_state(); + + if (!wnd) + return; + + if (st->hot_line0_dirty) { + wnd->hot_line0 = st->hot_line0_cache; + st->hot_line0_dirty = false; + } + if (st->gw_line1_dirty) { + memcpy(&wnd->gw_hot[0], &st->gw_hot_cache[0], 64); + st->gw_line1_dirty = false; + } + if (st->gw_line2_dirty) { + memcpy(&wnd->gw_hot[8], &st->gw_hot_cache[8], 64); + st->gw_line2_dirty = false; + } +#else + (void)wnd; +#endif +} + +/* Must be called with tel_lock held */ +static inline void telemetry_seq_begin(struct telemetry_wnd_data *wnd) +{ + wnd->ext_hdr.seq |= 1U; + compiler_barrier(); +} + +/* Must be called with tel_lock held */ +static inline void telemetry_seq_end(struct telemetry_wnd_data *wnd) +{ + compiler_barrier(); + wnd->ext_hdr.timestamp_us = (uint32_t)k_cyc_to_us_floor64(k_cycle_get_64()); + compiler_barrier(); + wnd->ext_hdr.seq = (wnd->ext_hdr.seq + 1U) & ~1U; +} + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES +/* Must be called with tel_lock held */ +static void telemetry_update_pipelines_section(struct telemetry_wnd_data *wnd) +{ + struct ipc *ipc = ipc_get(); + struct sof_tel_ppl_section *psec = &wnd->ppl_sec; + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0_locked(); + struct tel_ppl_slot new_slots[SOF_TEL_MAX_PIPELINES]; + struct list_item *clist; + k_spinlock_key_t ipc_key; + uint16_t ppl_cnt = 0; + uint16_t comp_cnt = 0; + + memset(new_slots, 0, sizeof(new_slots)); + memset(psec->core_sched, 0, sizeof(psec->core_sched)); + for (int c = 0; c < MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES); c++) { + psec->core_sched[c].core_id = (uint8_t)c; +#if CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL + if (cpu_is_core_enabled(c)) + psec->core_sched[c].active_kcps_budget = (uint32_t)core_kcps_get(c); +#endif + } + + if (!ipc) { + memset(tel_ppl_slots, 0, sizeof(tel_ppl_slots)); + if (hot0) + memset(hot0->ppl_xrun_count, 0, sizeof(hot0->ppl_xrun_count)); + psec->active_ppl_count = 0; + psec->total_comp_count = 0; + wnd->ext_hdr.secs[3].active_mask = 0; + return; + } + + ipc_key = k_spin_lock(&ipc->lock); + + /* First pass: collect pipelines */ + list_for_item(clist, &ipc->comp_list) { + struct ipc_comp_dev *icd = container_of(clist, struct ipc_comp_dev, list); + + if (icd->type == COMP_TYPE_COMPONENT) { + comp_cnt++; + if (icd->cd && icd->core < SOF_TEL_MAX_CORES) { + if (icd->cd->ipc_config.proc_domain == COMP_PROCESSING_DOMAIN_DP) + psec->core_sched[icd->core].dp_task_count++; + } + continue; + } + + if (icd->type != COMP_TYPE_PIPELINE || !icd->pipeline) + continue; + + if (ppl_cnt >= SOF_TEL_MAX_PIPELINES) + continue; + + struct pipeline *p = icd->pipeline; + struct sof_tel_ppl_info *pinfo = &psec->pipelines[ppl_cnt]; + uint16_t pid = (uint16_t)p->pipeline_id; + uint16_t prev_xruns = 0; + int32_t prev_xrun_bytes = 0; + + for (int s = 0; s < SOF_TEL_MAX_PIPELINES; s++) { + if (tel_ppl_slots[s].in_use && tel_ppl_slots[s].pipeline_id == pid) { + prev_xruns = tel_ppl_slots[s].xrun_count; + prev_xrun_bytes = tel_ppl_slots[s].last_xrun_bytes; + break; + } + } + + new_slots[ppl_cnt].in_use = true; + new_slots[ppl_cnt].pipeline_id = pid; + new_slots[ppl_cnt].xrun_count = prev_xruns; + new_slots[ppl_cnt].last_xrun_bytes = p->xrun_bytes ? p->xrun_bytes : + prev_xrun_bytes; + + pinfo->pipeline_id = pid; + pinfo->core = (uint8_t)p->core; + pinfo->priority = (uint8_t)p->priority; + pinfo->status = (uint8_t)p->status; + pinfo->comp_count = 0; + pinfo->attributes = (uint16_t)p->attributes; + pinfo->period_us = p->period; + pinfo->sched_comp_id = p->sched_id; + pinfo->last_xrun_bytes = new_slots[ppl_cnt].last_xrun_bytes; + + if (p->core < SOF_TEL_MAX_CORES) { + psec->core_sched[p->core].active_ppl_count++; + if (p->status == COMP_STATE_ACTIVE) + psec->core_sched[p->core].ll_task_count++; + } + ppl_cnt++; + } + + /* Clear unused pipeline slots */ + for (uint16_t i = ppl_cnt; i < SOF_TEL_MAX_PIPELINES; i++) + memset(&psec->pipelines[i], 0, sizeof(psec->pipelines[i])); + + /* Second pass: count components per pipeline */ + list_for_item(clist, &ipc->comp_list) { + struct ipc_comp_dev *icd = container_of(clist, struct ipc_comp_dev, list); + + if (icd->type != COMP_TYPE_COMPONENT || !icd->cd) + continue; + + uint32_t pid = dev_comp_pipe_id(icd->cd); + + for (uint16_t i = 0; i < ppl_cnt; i++) { + if (psec->pipelines[i].pipeline_id == (uint16_t)pid) { + psec->pipelines[i].comp_count++; + break; + } + } + } + + k_spin_unlock(&ipc->lock, ipc_key); + + memcpy(tel_ppl_slots, new_slots, sizeof(tel_ppl_slots)); + if (hot0) { + for (uint16_t i = 0; i < SOF_TEL_MAX_PIPELINES; i++) + hot0->ppl_xrun_count[i] = tel_ppl_slots[i].in_use ? + tel_ppl_slots[i].xrun_count : 0; + } + + for (int c = 0; c < MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES); c++) { + if (hot0) + hot0->active_ll_tasks[c] = psec->core_sched[c].ll_task_count; + if (psec->core_sched[c].ll_task_count == 0) { + wnd->system_tick_info[c].last_time_elapsed = 0; + wnd->system_tick_info[c].avg_utilization = 0; +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + telemetry_perf_period_sum[c] = 0; + telemetry_perf_period_cnt[c] = 0; + memset(&telemetry_perf_queue[c], 0, sizeof(telemetry_perf_queue[c])); +#endif + } + } + + psec->active_ppl_count = ppl_cnt; + psec->total_comp_count = comp_cnt; + wnd->ext_hdr.secs[3].active_mask = ppl_cnt; +} + +void telemetry_notify_pipeline_change(void) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + k_spinlock_key_t key; + + if (!telemetry_ext_ready || !wnd) + return; + + key = k_spin_lock(&tel_lock); + telemetry_seq_begin(wnd); + telemetry_update_pipelines_section(wnd); + telemetry_flush_hot_staging_locked(wnd); + telemetry_seq_end(wnd); + k_spin_unlock(&tel_lock, key); + + k_work_reschedule(&telemetry_bg_dwork, K_NO_WAIT); +} + +void telemetry_notify_pipeline_xrun(struct pipeline *p, int32_t bytes) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + k_spinlock_key_t key; + uint16_t pid; + int slot = -1; + int free_slot = -1; + + if (!telemetry_ext_ready || !wnd || !p) + return; + + pid = (uint16_t)p->pipeline_id; + key = k_spin_lock(&tel_lock); + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0_locked(); + + if (hot0) + hot0->total_xrun_events++; + + for (int i = 0; i < SOF_TEL_MAX_PIPELINES; i++) { + if (tel_ppl_slots[i].in_use && tel_ppl_slots[i].pipeline_id == pid) { + slot = i; + break; + } + if (!tel_ppl_slots[i].in_use && free_slot < 0) + free_slot = i; + } + + if (slot < 0 && free_slot >= 0) { + slot = free_slot; + tel_ppl_slots[slot].in_use = true; + tel_ppl_slots[slot].pipeline_id = pid; + tel_ppl_slots[slot].xrun_count = 0; + } + + if (slot >= 0) { + tel_ppl_slots[slot].xrun_count++; + tel_ppl_slots[slot].last_xrun_bytes = bytes; + if (hot0) + hot0->ppl_xrun_count[slot] = tel_ppl_slots[slot].xrun_count; + telemetry_seq_begin(wnd); + wnd->ppl_sec.pipelines[slot].pipeline_id = pid; + wnd->ppl_sec.pipelines[slot].last_xrun_bytes = bytes; + telemetry_seq_end(wnd); + } + k_spin_unlock(&tel_lock, key); +} + +void telemetry_notify_dp_deadline_miss(int core) +{ + k_spinlock_key_t key; + + if (!telemetry_ext_ready || core < 0 || core >= SOF_TEL_MAX_CORES) + return; + + key = k_spin_lock(&tel_lock); + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0_locked(); + + if (hot0) + hot0->dp_deadline_misses[core]++; + k_spin_unlock(&tel_lock, key); +} +#endif /* CONFIG_SOF_TELEMETRY_PIPELINES */ + +#ifdef CONFIG_SOF_TELEMETRY_DAIS +static uint8_t map_sof_dai_type(uint32_t type) +{ + switch (type) { + case SOF_DAI_INTEL_SSP: + return SOF_TEL_GW_SSP; + case SOF_DAI_INTEL_DMIC: + return SOF_TEL_GW_DMIC; + case SOF_DAI_INTEL_ALH: + return SOF_TEL_GW_ALH; + case SOF_DAI_INTEL_HDA: + return SOF_TEL_GW_HDA; + default: + return SOF_TEL_GW_OTHER; + } +} + +static void telemetry_fill_dai_cold(struct sof_tel_gw_cold *cold, + struct dai_data *dd, struct comp_dev *dev) +{ + memset(cold, 0, sizeof(*cold)); + cold->gw_type = map_sof_dai_type(dd->ipc_config.type); + cold->index = (uint8_t)dd->ipc_config.dai_index; + cold->direction = dev ? (uint8_t)dev->direction : (uint8_t)dd->ipc_config.direction; + cold->dma_chan = (int8_t)dd->chan_index; + cold->comp_id = dev ? (uint32_t)dev_comp_id(dev) : 0; + cold->state = dev ? (uint8_t)dev->state : 0; + cold->group_id = dd->group ? (uint8_t)dd->group->group_id : 0; + cold->period_bytes = (uint16_t)MIN(dd->period_bytes, UINT16_MAX); + cold->dma_buf_size = dd->dma_buffer ? + (uint32_t)audio_stream_get_size(&dd->dma_buffer->stream) : + dd->ipc_config.dma_buffer_size; +} + +static void telemetry_fill_host_cold(struct sof_tel_gw_cold *cold, + struct host_data *hd, struct comp_dev *dev) +{ + memset(cold, 0, sizeof(*cold)); + cold->gw_type = SOF_TEL_GW_HOST_DMA; + cold->index = (uint8_t)MAX(hd->chan_index, 0); + cold->direction = dev ? (uint8_t)dev->direction : 0; + cold->dma_chan = (int8_t)hd->chan_index; + cold->comp_id = dev ? (uint32_t)dev_comp_id(dev) : 0; + cold->state = dev ? (uint8_t)dev->state : 0; + cold->group_id = 0; + cold->period_bytes = (uint16_t)MIN(hd->period_bytes, UINT16_MAX); + cold->dma_buf_size = hd->dma_buffer ? + (uint32_t)audio_stream_get_size(&hd->dma_buffer->stream) : + hd->host_size; +} + +/* Must be called with tel_lock held */ +static void telemetry_sync_gw_cold_slot_locked(struct telemetry_wnd_data *wnd, int slot) +{ + uint16_t mask = 0; + + if (!telemetry_ext_ready || !wnd || slot < 0 || slot >= SOF_TEL_MAX_GATEWAYS) + return; + + struct sof_tel_gw_cold *cold = &wnd->dai_sec.gateways[slot]; + struct tel_gw_slot *s = &tel_gw_slots[slot]; + + if (s->in_use) + *cold = s->cold; + else + memset(cold, 0, sizeof(*cold)); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use) + mask |= BIT(i); + } + wnd->ext_hdr.secs[4].active_mask = mask; +} + +int telemetry_register_dai(struct dai_data *dd, struct comp_dev *dev) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + struct sof_tel_gw_cold cold; + k_spinlock_key_t key; + int free_slot = -1; + int slot = -1; + + if (!dd) + return -EINVAL; + + telemetry_fill_dai_cold(&cold, dd, dev); + key = k_spin_lock(&tel_lock); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && !tel_gw_slots[i].is_host && + tel_gw_slots[i].key == dd) { + slot = i; + break; + } + if (!tel_gw_slots[i].in_use && free_slot < 0) + free_slot = i; + } + + if (slot < 0) { + if (free_slot < 0) { + k_spin_unlock(&tel_lock, key); + return -ENOMEM; + } + slot = free_slot; + tel_gw_slots[slot].in_use = true; + tel_gw_slots[slot].is_host = false; + tel_gw_slots[slot].key = dd; + } + + tel_gw_slots[slot].cold = cold; + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot_locked(wnd, slot); + telemetry_seq_end(wnd); + } + k_spin_unlock(&tel_lock, key); + return slot; +} + +void telemetry_unregister_dai(struct dai_data *dd) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + k_spinlock_key_t key = k_spin_lock(&tel_lock); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && !tel_gw_slots[i].is_host && + tel_gw_slots[i].key == dd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot_locked(i); + + tel_gw_slots[i].in_use = false; + tel_gw_slots[i].key = NULL; + memset(&tel_gw_slots[i].cold, 0, sizeof(tel_gw_slots[i].cold)); + if (hot) + memset(hot, 0, sizeof(*hot)); + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot_locked(wnd, i); + telemetry_flush_hot_staging_locked(wnd); + telemetry_seq_end(wnd); + } + break; + } + } + k_spin_unlock(&tel_lock, key); +} + +void telemetry_update_dai_hot(struct dai_data *dd, struct comp_dev *dev) +{ + struct sof_tel_gw_cold cold; + uint32_t avail = 0; + bool has_avail = false; + k_spinlock_key_t key; + + if (!dd) + return; + + telemetry_fill_dai_cold(&cold, dd, dev); + if (dd->dma_buffer) { + avail = audio_stream_get_avail_bytes(&dd->dma_buffer->stream); + has_avail = true; + } + + key = k_spin_lock(&tel_lock); + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && !tel_gw_slots[i].is_host && + tel_gw_slots[i].key == dd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot_locked(i); + + tel_gw_slots[i].cold = cold; + if (!hot) + break; + + hot->total_kb_or_frames = (uint32_t)(dd->total_data_processed >> 10); + if (has_avail) + hot->dma_buf_avail_bytes = (uint16_t)MIN(avail, UINT16_MAX); + if (dd->xrun) + hot->xrun_or_skip_count++; + break; + } + } + k_spin_unlock(&tel_lock, key); +} + +int telemetry_register_host(struct host_data *hd, struct comp_dev *dev) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + struct sof_tel_gw_cold cold; + k_spinlock_key_t key; + int free_slot = -1; + int slot = -1; + + if (!hd) + return -EINVAL; + + telemetry_fill_host_cold(&cold, hd, dev); + key = k_spin_lock(&tel_lock); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + slot = i; + break; + } + if (!tel_gw_slots[i].in_use && free_slot < 0) + free_slot = i; + } + + if (slot < 0) { + if (free_slot < 0) { + k_spin_unlock(&tel_lock, key); + return -ENOMEM; + } + slot = free_slot; + tel_gw_slots[slot].in_use = true; + tel_gw_slots[slot].is_host = true; + tel_gw_slots[slot].key = hd; + } + + tel_gw_slots[slot].cold = cold; + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot_locked(wnd, slot); + telemetry_seq_end(wnd); + } + k_spin_unlock(&tel_lock, key); + return slot; +} + +void telemetry_unregister_host(struct host_data *hd) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + k_spinlock_key_t key = k_spin_lock(&tel_lock); + + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot_locked(i); + + tel_gw_slots[i].in_use = false; + tel_gw_slots[i].key = NULL; + memset(&tel_gw_slots[i].cold, 0, sizeof(tel_gw_slots[i].cold)); + if (hot) + memset(hot, 0, sizeof(*hot)); + if (telemetry_ext_ready && wnd) { + telemetry_seq_begin(wnd); + telemetry_sync_gw_cold_slot_locked(wnd, i); + telemetry_flush_hot_staging_locked(wnd); + telemetry_seq_end(wnd); + } + break; + } + } + k_spin_unlock(&tel_lock, key); +} + +void telemetry_update_host_hot(struct host_data *hd, struct comp_dev *dev) +{ + struct sof_tel_gw_cold cold; + uint32_t avail = 0; + bool has_avail = false; + k_spinlock_key_t key; + + if (!hd) + return; + + telemetry_fill_host_cold(&cold, hd, dev); + if (hd->local_buffer) { + avail = audio_stream_get_avail_bytes(&hd->local_buffer->stream); + has_avail = true; + } + + key = k_spin_lock(&tel_lock); + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) { + if (tel_gw_slots[i].in_use && tel_gw_slots[i].is_host && + tel_gw_slots[i].key == hd) { + struct sof_tel_gw_hot *hot = telemetry_get_gw_hot_locked(i); + + tel_gw_slots[i].cold = cold; + if (!hot) + break; + + hot->total_kb_or_frames = (uint32_t)(hd->total_data_processed >> 10); + if (has_avail) + hot->dma_buf_avail_bytes = (uint16_t)MIN(avail, UINT16_MAX); + if (hd->n_skipped) + hot->xrun_or_skip_count++; + break; + } + } + k_spin_unlock(&tel_lock, key); +} +#endif /* CONFIG_SOF_TELEMETRY_DAIS */ + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_notify_vregion_change(void) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + k_spinlock_key_t key; + + if (!telemetry_ext_ready || !wnd) + return; + + key = k_spin_lock(&tel_lock); + telemetry_seq_begin(wnd); + telemetry_collect_vpage_stats(&wnd->vreg_sec); + telemetry_collect_vregion_stats(&wnd->vreg_sec); + wnd->ext_hdr.secs[2].active_mask = wnd->vreg_sec.active_vregion_count; + telemetry_seq_end(wnd); + k_spin_unlock(&tel_lock, key); +} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_THREADS +extern int z_impl_k_thread_stack_space_get(const struct k_thread *thread, size_t *unused_ptr); + +#ifdef CONFIG_THREAD_RUNTIME_STATS +static uint64_t tel_prev_core_active[SOF_TEL_MAX_CORES]; +static uint64_t tel_prev_core_all[SOF_TEL_MAX_CORES]; +static uint32_t tel_last_core_delta_all[SOF_TEL_MAX_CORES]; +static struct { + const struct k_thread *tid; + uint64_t cycles; + uint32_t seen_seq; +} tel_prev_thread_cycles[SOF_TEL_MAX_THREADS]; +#endif + +static void telemetry_thread_cb(const struct k_thread *cthread, void *user_data) +{ + struct sof_tel_thread_section *tsec = user_data; + struct k_thread *thread = (struct k_thread *)cthread; + uint16_t idx = tsec->thread_count; + + if (idx >= SOF_TEL_MAX_THREADS) { + tsec->dropped_threads++; + return; + } + + struct sof_tel_thread_info *ti = &tsec->threads[idx]; + const char *name = k_thread_name_get((k_tid_t)thread); + + memset(ti, 0, sizeof(*ti)); + ti->thread_id = (uint32_t)POINTER_TO_UINT(thread); +#ifdef CONFIG_SMP + ti->core = thread->base.cpu; + if (thread->base.is_idle) + ti->flags |= SOF_TEL_THD_FLG_IDLE; +#endif + ti->prio = thread->base.prio; + ti->state = thread->base.thread_state; +#ifdef CONFIG_USERSPACE + if (thread->base.user_options & K_USER) + ti->flags |= SOF_TEL_THD_FLG_USER; +#endif + if (name) { + if (strncmp(name, "ll_thread", 9) == 0) + ti->flags |= SOF_TEL_THD_FLG_LL; + else if (strncmp(name, "DP_", 3) == 0 || strncmp(name, "edf_work", 8) == 0) + ti->flags |= SOF_TEL_THD_FLG_DP; + else if (strncmp(name, "idle", 4) == 0) + ti->flags |= SOF_TEL_THD_FLG_IDLE; + } + +#ifdef CONFIG_THREAD_STACK_INFO + if (thread->stack_info.size > 0) { + size_t unused = 0; + + ti->stack_size_chunks = (uint16_t)(thread->stack_info.size >> 6); + if (z_impl_k_thread_stack_space_get(thread, &unused) == 0 && + unused <= thread->stack_info.size) { + ti->stack_usage = (uint8_t)((255ULL * (thread->stack_info.size - unused)) / + thread->stack_info.size); + } + } +#endif + +#ifdef CONFIG_THREAD_RUNTIME_STATS + k_thread_runtime_stats_t rt_stats; + + if (k_thread_runtime_stats_get(thread, &rt_stats) == 0) { + uint32_t cur_seq = tsec->sample_seq + 1; + uint32_t delta_cyc = 0; + int empty_slot = -1; + int stale_slot = -1; + + ti->total_exec_kcycles = (uint32_t)(rt_stats.execution_cycles >> 10); + for (int i = 0; i < SOF_TEL_MAX_THREADS; i++) { + if (tel_prev_thread_cycles[i].tid == cthread) { + delta_cyc = (uint32_t)(rt_stats.execution_cycles - + tel_prev_thread_cycles[i].cycles); + tel_prev_thread_cycles[i].cycles = rt_stats.execution_cycles; + tel_prev_thread_cycles[i].seen_seq = cur_seq; + empty_slot = -2; + break; + } + if (empty_slot == -1 && !tel_prev_thread_cycles[i].tid) + empty_slot = i; + else if (stale_slot == -1 && + tel_prev_thread_cycles[i].seen_seq != cur_seq) + stale_slot = i; + } + if (empty_slot == -1) + empty_slot = stale_slot; + if (empty_slot >= 0) { + tel_prev_thread_cycles[empty_slot].tid = cthread; + tel_prev_thread_cycles[empty_slot].cycles = rt_stats.execution_cycles; + tel_prev_thread_cycles[empty_slot].seen_seq = cur_seq; + } + uint8_t c = ti->core < SOF_TEL_MAX_CORES ? ti->core : 0; + + if (tel_last_core_delta_all[c] > 0 && delta_cyc <= tel_last_core_delta_all[c]) { + ti->cpu_load = (uint8_t)((255ULL * delta_cyc) / + tel_last_core_delta_all[c]); + } + } +#endif + + if (name && name[0] != '\0') + strncpy(ti->name, name, sizeof(ti->name)); + + tsec->thread_count++; +} + +static void telemetry_update_threads_section(struct telemetry_wnd_data *wnd) +{ + struct sof_tel_thread_section *tsec = &wnd->thd_sec; + +#ifdef CONFIG_THREAD_RUNTIME_STATS + for (int c = 0; c < MIN(CONFIG_MAX_CORE_COUNT, SOF_TEL_MAX_CORES); c++) { + k_thread_runtime_stats_t core_stats; + + if (!cpu_is_core_enabled(c)) { + tsec->core_load[c] = 0; + tel_last_core_delta_all[c] = 0; + continue; + } + if (k_thread_runtime_stats_cpu_get(c, &core_stats) == 0) { + uint32_t act = (uint32_t)(core_stats.total_cycles - tel_prev_core_active[c]); + uint32_t all = (uint32_t)(core_stats.execution_cycles - tel_prev_core_all[c]); + + tel_prev_core_active[c] = core_stats.total_cycles; + tel_prev_core_all[c] = core_stats.execution_cycles; + tel_last_core_delta_all[c] = all; + tsec->core_load[c] = all ? (uint8_t)MIN(255ULL, (255ULL * act) / all) : 0; + } + } +#endif + + tsec->thread_count = 0; + tsec->dropped_threads = 0; + k_thread_foreach_unlocked(telemetry_thread_cb, tsec); + for (uint16_t i = tsec->thread_count; i < SOF_TEL_MAX_THREADS; i++) + memset(&tsec->threads[i], 0, sizeof(tsec->threads[i])); + tsec->sample_seq++; +#ifdef CONFIG_THREAD_RUNTIME_STATS + for (int i = 0; i < SOF_TEL_MAX_THREADS; i++) { + if (tel_prev_thread_cycles[i].tid && + tel_prev_thread_cycles[i].seen_seq != tsec->sample_seq) { + tel_prev_thread_cycles[i].tid = NULL; + tel_prev_thread_cycles[i].cycles = 0; + tel_prev_thread_cycles[i].seen_seq = 0; + } + } +#endif + wnd->ext_hdr.secs[5].active_mask = tsec->thread_count; +} +#endif /* CONFIG_SOF_TELEMETRY_THREADS */ + +void telemetry_refresh_cold_sections(void) +{ + struct telemetry_wnd_data *wnd = telemetry_get_wnd_ptr(); + k_spinlock_key_t key; + + if (!telemetry_ext_ready || !wnd) + return; + + key = k_spin_lock(&tel_lock); + telemetry_seq_begin(wnd); + +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + telemetry_collect_mem_stats(&wnd->mem_sec); + wnd->ext_hdr.secs[1].active_mask = SOF_TEL_MAX_HEAPS; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION + telemetry_collect_vpage_stats(&wnd->vreg_sec); + telemetry_collect_vregion_stats(&wnd->vreg_sec); + wnd->ext_hdr.secs[2].active_mask = wnd->vreg_sec.active_vregion_count; +#endif + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES + telemetry_update_pipelines_section(wnd); +#endif + +#ifdef CONFIG_SOF_TELEMETRY_DAIS + for (int i = 0; i < SOF_TEL_MAX_GATEWAYS; i++) + telemetry_sync_gw_cold_slot_locked(wnd, i); +#endif + +#ifdef CONFIG_SOF_TELEMETRY_THREADS + telemetry_update_threads_section(wnd); +#endif + + telemetry_flush_hot_staging_locked(wnd); + telemetry_seq_end(wnd); + k_spin_unlock(&tel_lock, key); +} + +static void telemetry_bg_work_handler(struct k_work *work) +{ + (void)work; + telemetry_refresh_cold_sections(); + if (telemetry_ext_ready) + k_work_reschedule(&telemetry_bg_dwork, TELEMETRY_BG_REFRESH_PERIOD); +} + +static void telemetry_ext_init(struct telemetry_wnd_data *wnd) +{ + struct sof_tel_ext_hdr *hdr = &wnd->ext_hdr; + + memset((uint8_t *)wnd + offsetof(struct telemetry_wnd_data, pad_to_0x100), 0, + sizeof(*wnd) - offsetof(struct telemetry_wnd_data, pad_to_0x100)); + + hdr->magic = SOF_TEL_EXT_MAGIC; + hdr->version = SOF_TEL_EXT_VERSION; + hdr->max_cores = CONFIG_MAX_CORE_COUNT; + hdr->num_secs = SOF_TEL_MAX_SECTIONS; + hdr->seq = 0; + + /* Section 0: Hot Counter Zone (0x0140, 256 B) */ + hdr->secs[0].id = SOF_TEL_SEC_HOT_ZONE; + hdr->secs[0].elem_size = 64; + hdr->secs[0].offset = (uint16_t)offsetof(struct telemetry_wnd_data, hot_line0); + hdr->secs[0].max_count = 4; + hdr->secs[0].active_mask = 3; + + /* Section 1: Memory & Heaps (0x0240, 256 B = 4 cache lines) */ + hdr->secs[1].id = SOF_TEL_SEC_MEMORY; + hdr->secs[1].elem_size = 64; + hdr->secs[1].offset = (uint16_t)offsetof(struct telemetry_wnd_data, mem_sec); + hdr->secs[1].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_MEMORY) ? 4 : 0; + + /* Section 2: vpage & vregion (0x0340, 512 B) */ + hdr->secs[2].id = SOF_TEL_SEC_VREGION; + hdr->secs[2].elem_size = sizeof(struct sof_tel_vregion_info); + hdr->secs[2].offset = (uint16_t)offsetof(struct telemetry_wnd_data, vreg_sec); + hdr->secs[2].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_VREGION) ? + SOF_TEL_MAX_VREGIONS : 0; + + /* Section 3: Pipelines & Schedulers (0x0540, 384 B) */ + hdr->secs[3].id = SOF_TEL_SEC_PIPELINES; + hdr->secs[3].elem_size = sizeof(struct sof_tel_ppl_info); + hdr->secs[3].offset = (uint16_t)offsetof(struct telemetry_wnd_data, ppl_sec); + hdr->secs[3].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_PIPELINES) ? + SOF_TEL_MAX_PIPELINES : 0; + + /* Section 4: DAIs & Host Gateways (0x06C0, 256 B) */ + hdr->secs[4].id = SOF_TEL_SEC_DAIS; + hdr->secs[4].elem_size = sizeof(struct sof_tel_gw_cold); + hdr->secs[4].offset = (uint16_t)offsetof(struct telemetry_wnd_data, dai_sec); + hdr->secs[4].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_DAIS) ? + SOF_TEL_MAX_GATEWAYS : 0; + + /* Section 5: Zephyr Threads (0x07C0, 832 B) */ + hdr->secs[5].id = SOF_TEL_SEC_THREADS; + hdr->secs[5].elem_size = sizeof(struct sof_tel_thread_info); + hdr->secs[5].offset = (uint16_t)offsetof(struct telemetry_wnd_data, thd_sec); + hdr->secs[5].max_count = IS_ENABLED(CONFIG_SOF_TELEMETRY_THREADS) ? + SOF_TEL_MAX_THREADS : 0; + + k_work_init_delayable(&telemetry_bg_dwork, telemetry_bg_work_handler); +} +#endif /* CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW */ + #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS static void telemetry_perf_queue_append(struct telemetry_perf_queue *q, size_t element) { @@ -130,12 +1091,17 @@ __cold int telemetry_init(void) systick_info[i].peak_utilization_8k = 0; } +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + telemetry_ext_init(wnd_data); +#endif + return 0; } void telemetry_update(uint32_t begin_stamp, uint32_t current_stamp) { int prid = cpu_get_id(); + uint32_t elapsed = current_stamp - begin_stamp; #ifdef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER struct system_tick_info *systick_info = telemetry_get_systick_info_ptr(); @@ -151,16 +1117,14 @@ void telemetry_update(uint32_t begin_stamp, uint32_t current_stamp) ++telemetry_systick_counter[prid]; systick_info[prid].count = telemetry_systick_counter[prid]; - systick_info[prid].last_time_elapsed = current_stamp - begin_stamp; + systick_info[prid].last_time_elapsed = elapsed; systick_info[prid].max_time_elapsed = - MAX(current_stamp - begin_stamp, - systick_info[prid].max_time_elapsed); + MAX(elapsed, systick_info[prid].max_time_elapsed); systick_info[prid].last_ccount = current_stamp; #ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS - const size_t measured_systick = begin_stamp - telemetry_prev_ccount[prid]; + const size_t measured_systick = elapsed; - telemetry_prev_ccount[prid] = begin_stamp; if (telemetry_systick_counter[prid] > 2) { telemetry_perf_period_sum[prid] += measured_systick; telemetry_perf_period_cnt[prid] = @@ -194,8 +1158,52 @@ void telemetry_update(uint32_t begin_stamp, uint32_t current_stamp) systick_info[prid].peak_utilization_8k = 0; } #endif + +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + if (telemetry_ext_ready) { + uint32_t ll_budget_cycles = (uint32_t)k_us_to_cyc_ceil32(LL_TIMER_PERIOD_US); + uint32_t ticks; + k_spinlock_key_t key = k_spin_lock(&tel_lock); + + if (elapsed > ll_budget_cycles && prid < SOF_TEL_MAX_CORES) { + struct sof_tel_hot_line0 *hot0 = telemetry_get_hot_line0_locked(); + + if (hot0) + hot0->ll_overrun_count[prid]++; + } + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES + if (prid < SOF_TEL_MAX_CORES && + wnd_data->ppl_sec.core_sched[prid].ll_task_count == 0) { + systick_info[prid].last_time_elapsed = 0; + systick_info[prid].avg_utilization = 0; + } +#endif + + telemetry_flush_hot_staging_locked(wnd_data); + ticks = ++tel_state()->total_tick_counter; + k_spin_unlock(&tel_lock, key); + + if ((ticks & 0x3ff) == 0) + k_work_reschedule(&telemetry_bg_dwork, K_NO_WAIT); + } +#endif } -/* init telemetry using Zephyr*/ -SYS_INIT(telemetry_init, APPLICATION, CONFIG_APPLICATION_INIT_PRIORITY); +/* + * Register the telemetry debug window slot before notify_host_boot_complete + * (POST_KERNEL, 0) sends IPC4_FW_READY, then refresh all cold sections once + * SOF and Zephyr application init (APPLICATION, 99) have completed. + */ +static int telemetry_late_init(void) +{ +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + telemetry_ext_ready = true; + telemetry_refresh_cold_sections(); + k_work_reschedule(&telemetry_bg_dwork, TELEMETRY_BG_REFRESH_PERIOD); +#endif + return 0; +} +SYS_INIT(telemetry_init, PRE_KERNEL_2, 0); +SYS_INIT(telemetry_late_init, APPLICATION, 99); diff --git a/src/include/sof/debug/telemetry/performance_monitor.h b/src/include/sof/debug/telemetry/performance_monitor.h index 68a1592ff40f..fe081c8bd4ef 100644 --- a/src/include/sof/debug/telemetry/performance_monitor.h +++ b/src/include/sof/debug/telemetry/performance_monitor.h @@ -23,6 +23,52 @@ (sys_cache_uncached_ptr_get((__sparse_force void __sparse_cache *) \ (WIN3_MBASE + WIN3_OFFSET)))) +struct comp_dev; + +/* Memory Window 3 extended partition constants (8,192 B window) */ +#define SOF_TEL_MW3_MAGIC 0x32574D50U /* "PMW2" */ +#define SOF_TEL_MW3_MOD_HOT_OFFSET 0x1080U +#define SOF_TEL_MW3_MOD_COLD_OFFSET 0x1200U +#define SOF_TEL_MW3_IO_PERF_OFFSET 0x1800U +#define SOF_TEL_MW3_HDR_OFFSET 0x1FC0U +#define SOF_TEL_MW3_MAX_IO_ITEMS 128U + +/* Dense hot per-component counter (4 B -> 16 components per 64B cache line) */ +struct sof_tel_mod_hot { + uint16_t cpc_violation_count; + uint16_t xrun_count; +} __packed; + +/* Cold per-component module resource record (16 B -> 4 components per 64B cache line) */ +struct sof_tel_mod_cold { + uint32_t resource_id; + uint32_t heap_usage; + uint32_t heap_high_water_mark; + uint32_t configured_cpc; +} __packed; + +/* MW3 partition footer directory at 0x1FC0 (64 B = 1 cache line) */ +struct sof_tel_mw3_hdr { + uint32_t magic; /* SOF_TEL_MW3_MAGIC ("PMW2") */ + uint16_t version; + uint16_t max_comp_entries; + uint16_t max_io_entries; + uint16_t mod_hot_offset; + uint16_t mod_cold_offset; + uint16_t io_perf_offset; + uint8_t rsvd[48]; +} __packed; + +#if IS_ENABLED(CONFIG_SOF_TELEMETRY_MODULE_RESOURCES) +void perf_data_update_module_resources(struct comp_dev *dev); +void perf_data_inc_cpc_violation(struct comp_dev *dev); +void perf_data_inc_xrun(struct comp_dev *dev); +#else +static inline void perf_data_update_module_resources(struct comp_dev *dev) { (void)dev; } +static inline void perf_data_inc_cpc_violation(struct comp_dev *dev) { (void)dev; } +static inline void perf_data_inc_xrun(struct comp_dev *dev) { (void)dev; } +#endif + #if IS_ENABLED(CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS) || defined(__DOXYGEN__) /** * \brief Initializer for component performance data record. diff --git a/src/include/sof/debug/telemetry/telemetry.h b/src/include/sof/debug/telemetry/telemetry.h index a3694735cb4f..bde3e674f147 100644 --- a/src/include/sof/debug/telemetry/telemetry.h +++ b/src/include/sof/debug/telemetry/telemetry.h @@ -13,10 +13,20 @@ #define __SOF_TELEMETRY_H__ #include +#include +#include +#include #ifdef __ZEPHYR__ +#include +#include #include #endif +struct pipeline; +struct dai_data; +struct host_data; +struct comp_dev; + #ifndef CONFIG_INTEL_ADSP_DEBUG_SLOT_MANAGER /** Slot index in memory window 2 (Debug Window) allocated for telemetry structures */ #define SOF_DW_TELEMETRY_SLOT 1 @@ -27,6 +37,48 @@ /** Measurement period in runs over which rolling average is computed */ #define SOF_AVG_PERF_MEAS_PERIOD 16 +/* Extended telemetry constants */ +#define SOF_TEL_EXT_MAGIC 0x324C4554U /* "TEL2" */ +#define SOF_TEL_EXT_VERSION 1 +#define SOF_TEL_EXT_HDR_OFFSET 0x0100U +#define SOF_TEL_MAX_CORES 5 +#define SOF_TEL_MAX_SECTIONS 6 +#define SOF_TEL_MAX_HEAPS 6 +#define SOF_TEL_MAX_VMH_BUNDLES 10 +#define SOF_TEL_MAX_VPAGE_ELEMS 16 +#define SOF_TEL_MAX_VREGIONS 16 +#define SOF_TEL_MAX_PIPELINES 16 +#define SOF_TEL_MAX_GATEWAYS 16 +#define SOF_TEL_MAX_THREADS 32 + +enum sof_tel_section_id { + SOF_TEL_SEC_HOT_ZONE = 1, /* 64B-aligned hot counters (sched, ppl, dai) */ + SOF_TEL_SEC_MEMORY = 2, /* Heaps, VMH, physical SRAM pages, fast_get */ + SOF_TEL_SEC_VREGION = 3, /* vpage allocator + active vregions */ + SOF_TEL_SEC_PIPELINES = 4, /* Cold pipeline & scheduler topology */ + SOF_TEL_SEC_DAIS = 5, /* Cold DAI & Host DMA gateway descriptors */ + SOF_TEL_SEC_THREADS = 6, /* Per-core Zephyr thread CPU & stack usage */ +}; + +enum sof_tel_heap_type { + SOF_TEL_HEAP_NONE = 0, + SOF_TEL_HEAP_SYS = 1, + SOF_TEL_HEAP_SHARED = 2, + SOF_TEL_HEAP_LL_USER = 3, + SOF_TEL_HEAP_L3_IMR = 4, + SOF_TEL_HEAP_DRV_USER = 5, +}; + +enum sof_tel_gw_type { + SOF_TEL_GW_NONE = 0, + SOF_TEL_GW_SSP = 1, + SOF_TEL_GW_DMIC = 2, + SOF_TEL_GW_ALH = 3, + SOF_TEL_GW_HDA = 4, + SOF_TEL_GW_HOST_DMA = 5, + SOF_TEL_GW_OTHER = 6, +}; + /** * \brief Per-core system tick and cycle execution metrics. * \ingroup telemetry_perf_api @@ -47,13 +99,234 @@ struct system_tick_info { uint32_t rsvd[2]; /**< Reserved for padding and future metrics */ } __packed; +/* Section descriptor in TEL2 directory header (8 bytes) */ +struct sof_tel_sec_desc { + uint8_t id; /* enum sof_tel_section_id */ + uint8_t elem_size; /* Size of each item in bytes (or 0 if composite) */ + uint16_t offset; /* 64B-aligned byte offset from start of MW2 Slot 1 (subtract 4 for /sys/kernel/debug/sof/exception) */ + uint16_t max_count; /* Allocated capacity in items */ + uint16_t active_mask; /* Bitmask or count of active entries */ +} __packed; + +/* Self-describing directory header at slot offset 0x0100 (64 bytes = 1 cache line) */ +struct sof_tel_ext_hdr { + uint32_t magic; /* SOF_TEL_EXT_MAGIC ("TEL2") */ + uint8_t version; /* SOF_TEL_EXT_VERSION */ + uint8_t max_cores; /* CONFIG_MAX_CORE_COUNT */ + uint16_t num_secs; /* Number of enabled sections */ + uint32_t seq; /* Odd while updating cold/bg snapshot, even when consistent */ + uint32_t timestamp_us; /* DSP uptime in us at last snapshot update */ + struct sof_tel_sec_desc secs[SOF_TEL_MAX_SECTIONS]; +} __packed; + +/* Hot Cache Line 0 at slot offset 0x0140 (64 bytes = 1 cache line) */ +struct sof_tel_hot_line0 { + uint16_t ll_overrun_count[SOF_TEL_MAX_CORES]; /* 10 B */ + uint16_t dp_deadline_misses[SOF_TEL_MAX_CORES]; /* 10 B */ + uint16_t active_ll_tasks[SOF_TEL_MAX_CORES]; /* 10 B */ + uint16_t total_xrun_events; /* 2 B */ + uint16_t ppl_xrun_count[SOF_TEL_MAX_PIPELINES]; /* 32 B */ +} __packed; + +/* Hot Gateway entry at slot offset 0x0180 (8 bytes -> 8 gateways per 64B cache line) */ +struct sof_tel_gw_hot { + uint32_t total_kb_or_frames; /* Rolling KB (bytes >> 10) processed */ + uint16_t dma_buf_avail_bytes; /* Live DMA buffer fill level (clamped to 64KB) */ + uint16_t xrun_or_skip_count; /* dd->xrun count or hd->n_skipped */ +} __packed; + +/* Physical SRAM & EBB summary (24 bytes) */ +struct sof_tel_sram_info { + uint16_t hp_sram_free_pages; + uint16_t hp_sram_total_pages; + uint16_t lp_sram_free_pages; + uint16_t lp_sram_total_pages; + uint32_t ebb_power_mask_lo; + uint32_t ebb_power_mask_hi; + uint32_t lpsram_ebb_mask; + uint32_t rsvd; +} __packed; + +/* Zephyr k_heap telemetry entry (20 bytes) */ +struct sof_tel_heap_info { + uint16_t heap_type; /* enum sof_tel_heap_type */ + uint16_t alloc_fail_count; + uint32_t total_bytes; + uint32_t allocated_bytes; + uint32_t free_bytes; + uint32_t max_allocated_bytes; +} __packed; + +/* Virtual Memory Heap (vmh_heap) block bundle entry (8 bytes) */ +struct sof_tel_vmh_bundle { + uint8_t blk_sz_shift; + uint8_t rsvd; + uint16_t total_blocks; + uint16_t used_blocks; + uint16_t max_blocks_or_oom; +} __packed; + +/* fast_get DRAM-to-SRAM cache pool summary (16 bytes) */ +struct sof_tel_fast_get_info { + uint16_t active_entries; + uint16_t fail_count; + uint32_t total_sram_bytes; + uint32_t peak_sram_bytes; + uint32_t total_get_calls; +} __packed; + +/* Memory & Heaps Section at slot offset 0x0240 (256 bytes = 4 cache lines) */ +struct sof_tel_mem_section { + struct sof_tel_sram_info sram; /* 24 B */ + struct sof_tel_heap_info heaps[SOF_TEL_MAX_HEAPS]; /* 120 B */ + uint32_t vmh_region_size; /* 4 B */ + uint16_t vmh_bundle_count; /* 2 B */ + uint16_t vmh_oom_total; /* 2 B */ + struct sof_tel_vmh_bundle vmh_bundles[SOF_TEL_MAX_VMH_BUNDLES]; /* 80 B */ + struct sof_tel_fast_get_info fast_get; /* 16 B */ + uint8_t pad[8]; /* 8 B */ +} __packed; + +/* vpage allocation element (4 bytes) */ +struct sof_tel_valloc_elem { + uint16_t pages; + uint16_t vpage; +} __packed; + +/* Active vregion instance telemetry (24 bytes) */ +struct sof_tel_vregion_info { + uint32_t base_va; + uint16_t total_pages; + uint8_t use_count; + uint8_t type_and_flags; + uint32_t lifetime_size_bytes; + uint32_t lifetime_used_bytes; + uint32_t interim_alloc_bytes; + uint16_t interim_free_kb; + uint16_t lifetime_free_count; +} __packed; + +/* Virtual Pages & vregion Section at slot offset 0x0340 (512 bytes = 8 cache lines) */ +struct sof_tel_vregion_section { + uint16_t total_pages; /* 2 B */ + uint16_t free_pages; /* 2 B */ + uint16_t min_free_pages; /* 2 B */ + uint16_t num_elems_in_use; /* 2 B */ + uint32_t alloc_fail_count; /* 4 B */ + uint32_t region_base_va; /* 4 B */ + struct sof_tel_valloc_elem velems[SOF_TEL_MAX_VPAGE_ELEMS]; /* 64 B */ + uint16_t active_vregion_count; /* 2 B */ + uint16_t total_vregions_created; /* 2 B */ + uint8_t pad[44]; /* 44 B */ + struct sof_tel_vregion_info vregions[SOF_TEL_MAX_VREGIONS]; /* 384 B */ +} __packed; + +/* Per-core scheduler summary (12 bytes) */ +struct sof_tel_core_sched { + uint8_t core_id; + uint8_t ll_task_count; + uint8_t dp_task_count; + uint8_t active_ppl_count; + uint32_t active_kcps_budget; + uint32_t rsvd; +} __packed; + +/* Cold pipeline descriptor (20 bytes) */ +struct sof_tel_ppl_info { + uint16_t pipeline_id; + uint8_t core; + uint8_t priority; + uint8_t status; + uint8_t comp_count; + uint16_t attributes; + uint32_t period_us; + uint32_t sched_comp_id; + int32_t last_xrun_bytes; +} __packed; + +/* Pipelines & Schedulers Section at slot offset 0x0540 (384 bytes = 6 cache lines) */ +struct sof_tel_ppl_section { + struct sof_tel_core_sched core_sched[SOF_TEL_MAX_CORES]; /* 60 B */ + uint16_t active_ppl_count; /* 2 B */ + uint16_t total_comp_count; /* 2 B */ + struct sof_tel_ppl_info pipelines[SOF_TEL_MAX_PIPELINES]; /* 320 B */ +} __packed; + +/* Cold DAI / Host DMA gateway descriptor (16 bytes) */ +struct sof_tel_gw_cold { + uint8_t gw_type; /* enum sof_tel_gw_type */ + uint8_t index; /* dai_index or stream_tag */ + uint8_t direction; /* PLAYBACK=0, CAPTURE=1 */ + int8_t dma_chan; /* chan_index */ + uint32_t comp_id; /* Owning Copier component ID */ + uint8_t state; /* COMP_STATE_* */ + uint8_t group_id; /* DAI group ID */ + uint16_t period_bytes; + uint32_t dma_buf_size; +} __packed; + +/* Cold DAIs & Host Gateways Section at slot offset 0x06C0 (256 bytes = 4 cache lines) */ +struct sof_tel_dai_section { + struct sof_tel_gw_cold gateways[SOF_TEL_MAX_GATEWAYS]; /* 256 B */ +} __packed; + +#define SOF_TEL_THD_FLG_IDLE BIT(0) +#define SOF_TEL_THD_FLG_USER BIT(1) +#define SOF_TEL_THD_FLG_DP BIT(2) +#define SOF_TEL_THD_FLG_LL BIT(3) + +/* Per-thread telemetry entry (24 bytes) */ +struct sof_tel_thread_info { + uint32_t thread_id; + uint8_t core; + int8_t prio; + uint8_t state; + uint8_t cpu_load; /* 0..255 scaled (0..100%) */ + uint8_t stack_usage; /* 0..255 scaled high-water mark (0..100%) */ + uint8_t flags; /* bit0=idle, bit1=user, bit2=dp, bit3=ll */ + uint16_t stack_size_chunks; /* stack size in 64-byte chunks */ + uint32_t total_exec_kcycles; /* execution_cycles >> 10 */ + char name[8]; /* 8-char thread name prefix */ +} __packed; + +/* Zephyr RTOS Threads Section at slot offset 0x07C0 (832 bytes = 13 cache lines) */ +struct sof_tel_thread_section { + uint8_t core_load[8]; /* 8 B */ + uint16_t thread_count; /* 2 B */ + uint16_t dropped_threads; /* 2 B */ + uint32_t sample_seq; /* 4 B */ + uint32_t rsvd[4]; /* 16 B */ + struct sof_tel_thread_info threads[SOF_TEL_MAX_THREADS];/* 768 B */ + uint8_t pad[32]; /* 32 B */ +} __packed; + /** - * \brief Layout of telemetry data published in shared Memory Window. + * \brief Layout of telemetry data published in shared Memory Window 2 Slot 1. * \ingroup telemetry_perf_api + * + * Legacy separator_1 + system_tick_info[] sits at 0x0000..0x00FF, + * followed at 64-byte-aligned offset 0x0100 by the TEL2 directory, + * Hot Counter Zone (0x0140..0x023F), and Cold Metadata Sections (0x0240..0x0AFF). */ struct telemetry_wnd_data { uint32_t separator_1; /**< Header boundary delimiter */ struct system_tick_info system_tick_info[CONFIG_MAX_CORE_COUNT]; /**< Per-core execution metrics */ +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW + uint8_t pad_to_0x100[SOF_TEL_EXT_HDR_OFFSET - sizeof(uint32_t) - + sizeof(struct system_tick_info) * CONFIG_MAX_CORE_COUNT]; + /* 0x0100: Self-describing directory header (64 B / 1 CL) */ + struct sof_tel_ext_hdr ext_hdr; + /* 0x0140: Hot Counter Zone (256 B / 4 CLs) */ + struct sof_tel_hot_line0 hot_line0; + struct sof_tel_gw_hot gw_hot[SOF_TEL_MAX_GATEWAYS]; + uint8_t hot_rsvd[64]; + /* 0x0240: Cold Metadata & Background Snapshot Zone */ + struct sof_tel_mem_section mem_sec; + struct sof_tel_vregion_section vreg_sec; + struct sof_tel_ppl_section ppl_sec; + struct sof_tel_dai_section dai_sec; + struct sof_tel_thread_section thd_sec; +#endif } __packed; /** @@ -85,6 +358,88 @@ void telemetry_update(uint32_t begin_ccount, uint32_t current_ccount); struct system_tick_info *telemetry_get_systick_info_ptr(void); #endif +#ifdef CONFIG_SOF_TELEMETRY_EXTENDED_WINDOW +struct telemetry_wnd_data *telemetry_get_wnd_ptr(void); +void telemetry_refresh_cold_sections(void); +#else +static inline struct telemetry_wnd_data *telemetry_get_wnd_ptr(void) { return NULL; } +static inline void telemetry_refresh_cold_sections(void) {} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +void telemetry_collect_mem_stats(struct sof_tel_mem_section *mem); +void telemetry_collect_fast_get_stats(struct sof_tel_fast_get_info *fg); +void telemetry_notify_mem_alloc_fail(uint16_t heap_type); +#else +static inline void telemetry_collect_mem_stats(struct sof_tel_mem_section *mem) { (void)mem; } +static inline void telemetry_collect_fast_get_stats(struct sof_tel_fast_get_info *fg) { (void)fg; } +static inline void telemetry_notify_mem_alloc_fail(uint16_t heap_type) { (void)heap_type; } +#endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_collect_vpage_stats(struct sof_tel_vregion_section *vsec); +void telemetry_collect_vregion_stats(struct sof_tel_vregion_section *vsec); +void telemetry_notify_vregion_change(void); +#else +static inline void telemetry_collect_vpage_stats(struct sof_tel_vregion_section *vsec) +{ + (void)vsec; +} +static inline void telemetry_collect_vregion_stats(struct sof_tel_vregion_section *vsec) +{ + (void)vsec; +} +static inline void telemetry_notify_vregion_change(void) {} +#endif + +#ifdef CONFIG_SOF_TELEMETRY_PIPELINES +void telemetry_notify_pipeline_change(void); +void telemetry_notify_pipeline_xrun(struct pipeline *p, int32_t bytes); +void telemetry_notify_dp_deadline_miss(int core); +#else +static inline void telemetry_notify_pipeline_change(void) {} +static inline void telemetry_notify_pipeline_xrun(struct pipeline *p, int32_t bytes) +{ + (void)p; + (void)bytes; +} +static inline void telemetry_notify_dp_deadline_miss(int core) { (void)core; } +#endif + +#ifdef CONFIG_SOF_TELEMETRY_DAIS +int telemetry_register_dai(struct dai_data *dd, struct comp_dev *dev); +void telemetry_unregister_dai(struct dai_data *dd); +void telemetry_update_dai_hot(struct dai_data *dd, struct comp_dev *dev); +int telemetry_register_host(struct host_data *hd, struct comp_dev *dev); +void telemetry_unregister_host(struct host_data *hd); +void telemetry_update_host_hot(struct host_data *hd, struct comp_dev *dev); +#else +static inline int telemetry_register_dai(struct dai_data *dd, struct comp_dev *dev) +{ + (void)dd; + (void)dev; + return -1; +} +static inline void telemetry_unregister_dai(struct dai_data *dd) { (void)dd; } +static inline void telemetry_update_dai_hot(struct dai_data *dd, struct comp_dev *dev) +{ + (void)dd; + (void)dev; +} +static inline int telemetry_register_host(struct host_data *hd, struct comp_dev *dev) +{ + (void)hd; + (void)dev; + return -1; +} +static inline void telemetry_unregister_host(struct host_data *hd) { (void)hd; } +static inline void telemetry_update_host_hot(struct host_data *hd, struct comp_dev *dev) +{ + (void)hd; + (void)dev; +} +#endif + #ifdef CONFIG_TIMING_FUNCTIONS /** Timestamp counter accessor using Zephyr timing functions */ #define telemetry_timestamp timing_counter_get diff --git a/src/ipc/ipc-helper.c b/src/ipc/ipc-helper.c index 3480f113a4a2..9b586e626957 100644 --- a/src/ipc/ipc-helper.c +++ b/src/ipc/ipc-helper.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -362,29 +363,35 @@ __cold int ipc_comp_free(struct ipc *ipc, uint32_t comp_id) /* * A completed pipeline stores raw comp_dev pointers in its - * source_comp/sink_comp/sched_comp fields. + * source_comp/sink_comp/sched_comp fields. Unlink icd under + * ipc->lock before freeing cd so concurrent readers of + * ipc->comp_list never observe a freed component. */ + struct comp_dev *cd = icd->cd; + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_for_item(clist, &ipc->comp_list) { struct ipc_comp_dev *ppl_icd = container_of(clist, struct ipc_comp_dev, list); - if (ppl_icd->type != COMP_TYPE_PIPELINE) + if (ppl_icd->type != COMP_TYPE_PIPELINE || !ppl_icd->pipeline) continue; - if (ppl_icd->pipeline->source_comp == icd->cd) + if (ppl_icd->pipeline->source_comp == cd) ppl_icd->pipeline->source_comp = NULL; - if (ppl_icd->pipeline->sink_comp == icd->cd) + if (ppl_icd->pipeline->sink_comp == cd) ppl_icd->pipeline->sink_comp = NULL; - if (ppl_icd->pipeline->sched_comp == icd->cd) + if (ppl_icd->pipeline->sched_comp == cd) ppl_icd->pipeline->sched_comp = NULL; } - /* free component and remove from list */ - comp_free(icd->cd); - icd->cd = NULL; - list_item_del(&icd->list); + k_spin_unlock(&ipc->lock, key); + + /* free component and container */ + comp_free(cd); sof_heap_free(sof_sys_user_heap_get(), icd); + telemetry_notify_pipeline_change(); return 0; } diff --git a/src/ipc/ipc3/helper.c b/src/ipc/ipc3/helper.c index 2fb7b52e1f4a..52846fb33c05 100644 --- a/src/ipc/ipc3/helper.c +++ b/src/ipc/ipc3/helper.c @@ -10,7 +10,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -433,7 +435,11 @@ int ipc_pipeline_new(struct ipc *ipc, ipc_pipe_new *_pipe_desc) ipc_pipe->id = pipe_desc->comp_id; /* add new pipeline to the list */ + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_item_append(&ipc_pipe->list, &ipc->comp_list); + k_spin_unlock(&ipc->lock, key); + telemetry_notify_pipeline_change(); return 0; } @@ -443,6 +449,8 @@ int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) struct ipc_comp_dev *ipc_pipe; struct ipc_comp_dev *icd; struct list_item *clist; + struct pipeline *pipe; + k_spinlock_key_t key; int ret; /* check whether pipeline exists */ @@ -467,22 +475,27 @@ int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) * (or in fuzz/error paths) the dangling pointer would be a * use-after-free on any subsequent IPC referencing that component. */ + pipe = ipc_pipe->pipeline; + key = k_spin_lock(&ipc->lock); list_for_item(clist, &ipc->comp_list) { icd = container_of(clist, struct ipc_comp_dev, list); if (icd->type == COMP_TYPE_COMPONENT && - icd->cd && icd->cd->pipeline == ipc_pipe->pipeline) + icd->cd && icd->cd->pipeline == pipe) icd->cd->pipeline = NULL; } + ipc_pipe->pipeline = NULL; + list_item_del(&ipc_pipe->list); + k_spin_unlock(&ipc->lock, key); /* free buffer and remove from list */ - ret = pipeline_free(ipc_pipe->pipeline); + ret = pipeline_free(pipe); if (ret < 0) { tr_err(&ipc_tr, "pipeline_free() failed"); + rfree(ipc_pipe); return ret; } - ipc_pipe->pipeline = NULL; - list_item_del(&ipc_pipe->list); rfree(ipc_pipe); + telemetry_notify_pipeline_change(); return 0; } @@ -520,7 +533,10 @@ int ipc_buffer_new(struct ipc *ipc, const struct sof_ipc_buffer *desc) ibd->id = desc->comp.id; /* add new buffer to the list */ + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_item_append(&ibd->list, &ipc->comp_list); + k_spin_unlock(&ipc->lock, key); return ret; } @@ -608,8 +624,14 @@ int ipc_buffer_free(struct ipc *ipc, uint32_t buffer_id) pipeline_disconnect(source, ibd->cb, PPL_CONN_DIR_COMP_TO_BUFFER); /* free buffer and remove from list */ - buffer_free(ibd->cb); + struct comp_buffer *cb = ibd->cb; + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + + ibd->cb = NULL; list_item_del(&ibd->list); + k_spin_unlock(&ipc->lock, key); + + buffer_free(cb); rfree(ibd); return 0; @@ -722,7 +744,11 @@ int ipc_comp_new(struct ipc *ipc, ipc_comp *_comp) icd->id = comp->id; /* add new component to the list */ + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_item_append(&icd->list, &ipc->comp_list); + k_spin_unlock(&ipc->lock, key); + telemetry_notify_pipeline_change(); return 0; } diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d5a0e095b3f4..63d3893c4f12 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -523,7 +524,11 @@ __cold static int ipc4_create_pipeline(struct ipc4_pipeline_create *pipe_desc, ipc_pipe->pipeline->attributes = pipe_desc->extension.r.attributes; /* add new pipeline to the list */ + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_item_append(&ipc_pipe->list, &ipc->comp_list); + k_spin_unlock(&ipc->lock, key); + telemetry_notify_pipeline_change(); return IPC4_SUCCESS; } @@ -678,16 +683,26 @@ __cold int ipc_pipeline_free(struct ipc *ipc, uint32_t comp_id) return ret; } - /* free buffer, delete all tasks and remove from list */ - ret = pipeline_free(ipc_pipe->pipeline); + /* Unlink from ipc->comp_list before freeing pipeline so concurrent + * readers never observe a freed pipeline pointer. + */ + struct pipeline *pipe = ipc_pipe->pipeline; + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + + ipc_pipe->pipeline = NULL; + list_item_del(&ipc_pipe->list); + k_spin_unlock(&ipc->lock, key); + + /* free buffer, delete all tasks and free pipeline */ + ret = pipeline_free(pipe); if (ret < 0) { tr_err(&ipc_tr, "pipeline_free() failed"); + sof_heap_free(sof_sys_user_heap_get(), ipc_pipe); return IPC4_INVALID_RESOURCE_STATE; } - ipc_pipe->pipeline = NULL; - list_item_del(&ipc_pipe->list); sof_heap_free(sof_sys_user_heap_get(), ipc_pipe); + telemetry_notify_pipeline_change(); return IPC4_SUCCESS; } @@ -1238,14 +1253,20 @@ __cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dm if (ret < 0) return ret; + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_for_item_safe(clist, _tmp, &ipc->comp_list) { icd = container_of(clist, struct ipc_comp_dev, list); if (icd->cd != dev) continue; + icd->cd = NULL; list_item_del(&icd->list); + k_spin_unlock(&ipc->lock, key); sof_heap_free(sof_sys_user_heap_get(), icd); - break; + goto out_free_dev; } + k_spin_unlock(&ipc->lock, key); +out_free_dev: comp_free(dev); } return ret; @@ -1546,7 +1567,11 @@ __cold int ipc4_add_comp_dev(struct comp_dev *dev) tr_dbg(&ipc_tr, "add comp 0x%x", icd->id); /* add new component to the list */ + k_spinlock_key_t key = k_spin_lock(&ipc->lock); + list_item_append(&icd->list, &ipc->comp_list); + k_spin_unlock(&ipc->lock, key); + telemetry_notify_pipeline_change(); return IPC4_SUCCESS; }; diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 5e69d0943419..bda09fc55573 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -408,6 +409,15 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) /* get module deadline in us */ uint32_t deadline = module_get_deadline(mod); + if (!trigger_task && !mod->dp_startup_delay && + pdata->deadline_clock_ticks && + (deadline == 0 || + (int32_t)(dp_sch->last_ll_tick_timestamp - + pdata->deadline_clock_ticks) > 0)) { + telemetry_notify_dp_deadline_miss(curr_task->core); + pdata->deadline_clock_ticks = 0; + } + /* if a deadline cannot be calculated, use a fixed value relative to its * first start */ @@ -423,6 +433,8 @@ void scheduler_dp_recalculate(struct scheduler_dp_data *dp_sch) /* add to "NOW", overflows are OK */ deadline = dp_sch->last_ll_tick_timestamp + deadline; + if (trigger_task) + pdata->deadline_clock_ticks = deadline ? deadline : 1U; /* set in Zephyr. Note that it may be in past, it does not matter, * Zephyr still will schedule the thread with earlier deadline diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index fa8583832a03..7a4c4d9835de 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -82,6 +83,15 @@ static void scheduler_dp_recalculate_thread(struct scheduler_dp_data *dp_sch, bo /* get module deadline in us*/ uint32_t deadline = module_get_deadline(mod); + if (!trigger_task && is_ll_post_run && !mod->dp_startup_delay && + pdata->deadline_clock_ticks && + (deadline == 0 || + (int32_t)(dp_sch->last_ll_tick_timestamp - + pdata->deadline_clock_ticks) > 0)) { + telemetry_notify_dp_deadline_miss(curr_task->core); + pdata->deadline_clock_ticks = 0; + } + /* if a deadline cannot be calculated, use a fixed value relative to its * first start */ @@ -97,6 +107,8 @@ static void scheduler_dp_recalculate_thread(struct scheduler_dp_data *dp_sch, bo /* add to "NOW", overflows are OK */ deadline = dp_sch->last_ll_tick_timestamp + deadline; + if (trigger_task) + pdata->deadline_clock_ticks = deadline ? deadline : 1U; /* set in Zephyr. Note that it may be in past, it does not matter, * Zephyr still will schedule the thread with earlier deadline @@ -156,6 +168,14 @@ void dp_thread_fn(void *p1, void *p2, void *p3) * if not, set the state returned by run procedure */ if (task->state == SOF_TASK_STATE_RUNNING) { + if (!(task->flags & K_USER) && + !task_pdata->mod->dp_startup_delay && + task_pdata->deadline_clock_ticks && + (int32_t)(k_cycle_get_32() - + task_pdata->deadline_clock_ticks) > 0) { + telemetry_notify_dp_deadline_miss(task->core); + } + task_pdata->deadline_clock_ticks = 0; task->state = state; switch (state) { case SOF_TASK_STATE_RESCHEDULE: diff --git a/zephyr/include/sof/lib/regions_mm.h b/zephyr/include/sof/lib/regions_mm.h index 259091ab4df3..fbbea048a8c8 100644 --- a/zephyr/include/sof/lib/regions_mm.h +++ b/zephyr/include/sof/lib/regions_mm.h @@ -133,6 +133,10 @@ void vmh_log_stats(struct vmh_heap *heap); */ void vmh_log_alloc_stats(struct vmh_heap *heap); #endif +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +struct sof_tel_mem_section; +void vmh_fill_telemetry_stats(struct vmh_heap *heap, struct sof_tel_mem_section *mem); +#endif /** * @brief Checks if ptr is in range of given memory range. * diff --git a/zephyr/lib/alloc.c b/zephyr/lib/alloc.c index 89bd49286831..e486c0951c90 100644 --- a/zephyr/lib/alloc.c +++ b/zephyr/lib/alloc.c @@ -13,6 +13,7 @@ #include #include #include +#include #include /* for zephyr_ll_user_heap() */ #include #include @@ -411,6 +412,10 @@ struct k_heap *sof_sys_user_heap_get(void) } #if !defined(CONFIG_SOF_NATIVE_SIM_HOST_HEAP) +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +static uint16_t heap_fail_counts[SOF_TEL_MAX_HEAPS]; +#endif + static void *heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes) { k_spinlock_key_t key; @@ -437,6 +442,19 @@ static void *heap_alloc_aligned(struct k_heap *h, size_t min_align, size_t bytes ret = sys_heap_aligned_alloc(&h->heap, min_align, bytes); k_spin_unlock(&h->lock, key); +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + if (!ret) { + if (h == &sof_heap) + heap_fail_counts[0]++; +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP + else if (h == &shared_buffer_heap) + heap_fail_counts[1]++; +#endif + else + heap_fail_counts[4]++; + } +#endif + #if CONFIG_SYS_HEAP_RUNTIME_STATS && CONFIG_IPC_MAJOR_4 sys_heap_runtime_stats_get(&h->heap, &stats); tr_info(&zephyr_tr, "heap allocated: %u free: %u max allocated: %u", @@ -910,4 +928,120 @@ __weak void *_sbrk_r(struct _reent *ptr, ptrdiff_t incr) return NULL; } +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +static void fill_one_heap_telemetry(struct sof_tel_heap_info *dst, struct k_heap *h, + uint16_t type, uint16_t fail_cnt) +{ + dst->heap_type = type; + dst->alloc_fail_count = fail_cnt; + if (!h || !h->heap.heap) { + dst->total_bytes = 0; + dst->allocated_bytes = 0; + dst->free_bytes = 0; + dst->max_allocated_bytes = 0; + return; + } + + dst->total_bytes = (uint32_t)h->heap.init_bytes; +#if CONFIG_SYS_HEAP_RUNTIME_STATS + struct sys_memory_stats stats = { 0 }; + k_spinlock_key_t key = k_spin_lock(&h->lock); + int ret = sys_heap_runtime_stats_get(&h->heap, &stats); + + k_spin_unlock(&h->lock, key); + if (ret == 0) { + dst->allocated_bytes = (uint32_t)stats.allocated_bytes; + dst->free_bytes = (uint32_t)stats.free_bytes; + dst->max_allocated_bytes = (uint32_t)stats.max_allocated_bytes; + if (dst->total_bytes == 0) + dst->total_bytes = dst->allocated_bytes + dst->free_bytes; + } +#endif +} + +void telemetry_collect_mem_stats(struct sof_tel_mem_section *mem) +{ + uint32_t free_heap_pages; + + if (!mem) + return; + + fill_one_heap_telemetry(&mem->heaps[0], &sof_heap, + SOF_TEL_HEAP_SYS, heap_fail_counts[0]); +#if CONFIG_SOF_USERSPACE_USE_SHARED_HEAP + fill_one_heap_telemetry(&mem->heaps[1], &shared_buffer_heap, + SOF_TEL_HEAP_SHARED, heap_fail_counts[1]); +#endif +#ifdef CONFIG_SOF_USERSPACE_LL + fill_one_heap_telemetry(&mem->heaps[2], zephyr_ll_user_heap(), + SOF_TEL_HEAP_LL_USER, heap_fail_counts[2]); +#endif +#if CONFIG_L3_HEAP + fill_one_heap_telemetry(&mem->heaps[3], &l3_heap, + SOF_TEL_HEAP_L3_IMR, heap_fail_counts[3]); +#endif + + free_heap_pages = (mem->heaps[0].free_bytes + mem->heaps[1].free_bytes + + mem->heaps[2].free_bytes) / HOST_PAGE_SIZE; + +#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) + uint32_t hp_banks = ace_hpsram_get_bank_count(); + uint32_t lp_banks = ace_lpsram_get_bank_count(); + uint64_t hp_mask = 0; + uint32_t lp_mask = 0; + uint32_t active_hp_banks = 0; + uint32_t active_lp_banks = 0; + uint32_t unpowered_hp_pages = 0; + uint32_t unpowered_lp_pages = 0; + + for (uint32_t i = 0; i < MIN(hp_banks, 64U); i++) { + if (HPSRAM_REGS(i)->HSxPGISTS == 0) { + hp_mask |= BIT64(i); + active_hp_banks++; + } + } + for (uint32_t i = 0; i < MIN(lp_banks, 32U); i++) { + if (LPSRAM_REGS(i)->USxPGISTS == 0) { + lp_mask |= BIT(i); + active_lp_banks++; + } + } + mem->sram.hp_sram_total_pages = (uint16_t)(L2_SRAM_SIZE / HOST_PAGE_SIZE); + if (hp_banks && hp_banks >= active_hp_banks) + unpowered_hp_pages = + (mem->sram.hp_sram_total_pages * (hp_banks - active_hp_banks)) / hp_banks; + mem->sram.hp_sram_free_pages = + (uint16_t)MIN((uint32_t)mem->sram.hp_sram_total_pages, + unpowered_hp_pages + free_heap_pages); + + mem->sram.lp_sram_total_pages = (uint16_t)(LP_SRAM_SIZE / HOST_PAGE_SIZE); + if (lp_banks && lp_banks >= active_lp_banks) + unpowered_lp_pages = + (mem->sram.lp_sram_total_pages * (lp_banks - active_lp_banks)) / lp_banks; + mem->sram.lp_sram_free_pages = (uint16_t)unpowered_lp_pages; + mem->sram.ebb_power_mask_lo = (uint32_t)(hp_mask & 0xFFFFFFFFU); + mem->sram.ebb_power_mask_hi = (uint32_t)(hp_mask >> 32); + mem->sram.lpsram_ebb_mask = lp_mask; +#elif defined(PLATFORM_HPSRAM_EBB_COUNT) + mem->sram.hp_sram_total_pages = + (uint16_t)((SRAM_BANK_SIZE * PLATFORM_HPSRAM_EBB_COUNT) / HOST_PAGE_SIZE); + mem->sram.hp_sram_free_pages = + (uint16_t)MIN((uint32_t)mem->sram.hp_sram_total_pages, free_heap_pages); + mem->sram.ebb_power_mask_lo = HPSRAM_REGS(0)->HSxPGCTL; +#if PLATFORM_HPSRAM_EBB_COUNT > 32 + mem->sram.ebb_power_mask_hi = HPSRAM_REGS(1)->HSxPGCTL; +#endif +#endif + +#if CONFIG_VIRTUAL_HEAP + if (virtual_buffers_heap) + vmh_fill_telemetry_stats(virtual_buffers_heap, mem); +#endif + +#if defined(CONFIG_FAST_GET) && defined(CONFIG_COLD_STORE_EXECUTE_DRAM) + telemetry_collect_fast_get_stats(&mem->fast_get); +#endif +} +#endif + SYS_INIT(heap_init, PRE_KERNEL_1, CONFIG_KERNEL_INIT_PRIORITY_OBJECTS); diff --git a/zephyr/lib/fast-get.c b/zephyr/lib/fast-get.c index 24d6af4750b5..05bea485323f 100644 --- a/zephyr/lib/fast-get.c +++ b/zephyr/lib/fast-get.c @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -37,6 +38,12 @@ static struct sof_fast_get_data fast_get_data = { .pool.list = LIST_INIT(fast_get_data.pool.list), }; +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +static uint32_t fast_get_total_calls; +static uint16_t fast_get_fail_count; +static uint32_t fast_get_peak_bytes; +#endif + LOG_MODULE_REGISTER(fast_get, CONFIG_SOF_LOG_LEVEL); struct fast_get_find { @@ -271,6 +278,11 @@ const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t s entry->sram_ptr = ret; entry->refcount = 1; out: +#ifdef CONFIG_SOF_TELEMETRY_MEMORY + fast_get_total_calls++; + if (!ret) + fast_get_fail_count++; +#endif k_spin_unlock(&data->lock, key); LOG_DBG("get %p, %p, size %u, refcnt %u", dram_ptr, ret, size, entry ? entry->refcount : 0); @@ -278,6 +290,46 @@ const void *fast_get(struct mod_alloc_ctx *alloc, const void *dram_ptr, size_t s } EXPORT_SYMBOL(fast_get); +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +struct fast_get_tel_acc { + uint16_t active_entries; + uint32_t total_bytes; +}; + +static bool fast_get_tel_cb(void *data, void *arg) +{ + struct sof_fast_get_entry *entry = data; + struct fast_get_tel_acc *acc = arg; + + if (entry->sram_ptr && entry->refcount > 0) { + acc->active_entries++; + acc->total_bytes += (uint32_t)entry->size; + } + return false; +} + +void telemetry_collect_fast_get_stats(struct sof_tel_fast_get_info *fg) +{ + struct fast_get_tel_acc acc = { 0 }; + k_spinlock_key_t key; + + if (!fg) + return; + + key = k_spin_lock(&fast_get_data.lock); + objpool_iterate(&fast_get_data.pool, fast_get_tel_cb, &acc); + if (acc.total_bytes > fast_get_peak_bytes) + fast_get_peak_bytes = acc.total_bytes; + + fg->active_entries = acc.active_entries; + fg->fail_count = fast_get_fail_count; + fg->total_sram_bytes = acc.total_bytes; + fg->peak_sram_bytes = fast_get_peak_bytes; + fg->total_get_calls = fast_get_total_calls; + k_spin_unlock(&fast_get_data.lock, key); +} +#endif + static bool fast_put_find_entry(void *data, void *arg) { struct sof_fast_get_entry *entry = data; diff --git a/zephyr/lib/regions_mm.c b/zephyr/lib/regions_mm.c index 24d642cd0903..a7f505691b6b 100644 --- a/zephyr/lib/regions_mm.c +++ b/zephyr/lib/regions_mm.c @@ -10,6 +10,9 @@ #if defined(CONFIG_MM_DRV) #include #include +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +#include +#endif LOG_MODULE_DECLARE(mem_allocator, CONFIG_SOF_LOG_LEVEL); @@ -797,4 +800,47 @@ void vmh_get_default_heap_config(const struct sys_mm_drv_region *region, } } +#ifdef CONFIG_SOF_TELEMETRY_MEMORY +void vmh_fill_telemetry_stats(struct vmh_heap *heap, struct sof_tel_mem_section *mem) +{ + uint16_t bundle_cnt = 0; + uint16_t oom_total = 0; + + if (!heap || !mem) + return; + + mem->vmh_region_size = heap->virtual_region ? (uint32_t)heap->virtual_region->size : 0; + + for (int idx = 0; idx < MAX_MEMORY_ALLOCATORS_COUNT && + bundle_cnt < SOF_TEL_MAX_VMH_BUNDLES; idx++) { + struct sys_mem_blocks *blk = heap->physical_blocks_allocators[idx]; + uint32_t used = 0; + + if (!blk) + continue; + + if (blk->bitmap && blk->bitmap->bundles) { + for (size_t k = 0; k < blk->bitmap->num_bundles; k++) + used += POPCOUNT(blk->bitmap->bundles[k]); + } + + mem->vmh_bundles[bundle_cnt].blk_sz_shift = blk->info.blk_sz_shift; + mem->vmh_bundles[bundle_cnt].total_blocks = (uint16_t)blk->info.num_blocks; + mem->vmh_bundles[bundle_cnt].used_blocks = (uint16_t)used; +#ifdef CONFIG_SYS_MEM_BLOCKS_RUNTIME_STATS + mem->vmh_bundles[bundle_cnt].max_blocks_or_oom = + (uint16_t)blk->info.max_used_blocks; + oom_total += (uint16_t)heap->out_of_blocks[idx]; +#else + if ((uint16_t)used > mem->vmh_bundles[bundle_cnt].max_blocks_or_oom) + mem->vmh_bundles[bundle_cnt].max_blocks_or_oom = (uint16_t)used; +#endif + bundle_cnt++; + } + + mem->vmh_bundle_count = bundle_cnt; + mem->vmh_oom_total = oom_total; +} +#endif + #endif /* if defined (CONFIG_MM_DRV) */ diff --git a/zephyr/lib/vpage.c b/zephyr/lib/vpage.c index 8a7e54261006..2b89bac101e8 100644 --- a/zephyr/lib/vpage.c +++ b/zephyr/lib/vpage.c @@ -12,6 +12,7 @@ #include #include #include +#include #include LOG_MODULE_REGISTER(vpage, CONFIG_SOF_LOG_LEVEL); @@ -63,6 +64,43 @@ struct vpage_context { /* uncache persistent across all cores */ static struct vpage_context vpage_ctx; +#ifdef CONFIG_SOF_TELEMETRY_VREGION +static struct k_spinlock vpage_tel_lock; +static struct sof_tel_vregion_section vpage_tel_snap; +static uint16_t vpage_min_free_pages = UINT16_MAX; +static uint32_t vpage_alloc_fail_count; + +/* Must be called with vpage_ctx.lock held (or during init) */ +static void vpage_tel_sync_locked(void) +{ + k_spinlock_key_t key = k_spin_lock(&vpage_tel_lock); + + if (vpage_ctx.virtual_region) { + vpage_tel_snap.total_pages = (uint16_t)vpage_ctx.total_pages; + vpage_tel_snap.free_pages = (uint16_t)vpage_ctx.free_pages; + vpage_tel_snap.min_free_pages = (vpage_min_free_pages == UINT16_MAX) ? + (uint16_t)vpage_ctx.free_pages : + vpage_min_free_pages; + vpage_tel_snap.num_elems_in_use = (uint16_t)vpage_ctx.num_elems_in_use; + vpage_tel_snap.alloc_fail_count = vpage_alloc_fail_count; + vpage_tel_snap.region_base_va = + (uint32_t)POINTER_TO_UINT(vpage_ctx.virtual_region->addr); + + for (int i = 0; i < SOF_TEL_MAX_VPAGE_ELEMS; i++) { + if (i < VPAGE_MAX_ALLOCS) { + vpage_tel_snap.velems[i].pages = vpage_ctx.velems[i].pages; + vpage_tel_snap.velems[i].vpage = vpage_ctx.velems[i].vpage; + } else { + vpage_tel_snap.velems[i].pages = 0; + vpage_tel_snap.velems[i].vpage = 0; + } + } + } + k_spin_unlock(&vpage_tel_lock, key); +} +#else +static inline void vpage_tel_sync_locked(void) {} +#endif /** * @brief Allocate and map virtual memory pages @@ -159,6 +197,13 @@ __cold void *vpage_alloc(unsigned int pages) k_mutex_lock(&vpage_ctx.lock, K_FOREVER); ret = vpages_alloc_and_map(pages, &ptr); +#ifdef CONFIG_SOF_TELEMETRY_VREGION + if (ret < 0) + vpage_alloc_fail_count++; + else if (vpage_ctx.free_pages < vpage_min_free_pages) + vpage_min_free_pages = (uint16_t)vpage_ctx.free_pages; + vpage_tel_sync_locked(); +#endif k_mutex_unlock(&vpage_ctx.lock); if (ret < 0) LOG_ERR("vpage_alloc failed %d for %d pages, total %d free %d", @@ -166,6 +211,7 @@ __cold void *vpage_alloc(unsigned int pages) else LOG_INF("vpage_alloc ptr %p pages %u free %u/%u", ptr, pages, vpage_ctx.free_pages, vpage_ctx.total_pages); + telemetry_notify_vregion_change(); return ptr; } @@ -256,12 +302,36 @@ void vpage_free(void *ptr) k_mutex_lock(&vpage_ctx.lock, K_FOREVER); ret = vpages_free_and_unmap((uintptr_t *)ptr); + if (!ret) + vpage_tel_sync_locked(); k_mutex_unlock(&vpage_ctx.lock); - if (!ret) + if (!ret) { LOG_INF("vptr %p free/total pages %d/%d", ptr, vpage_ctx.free_pages, vpage_ctx.total_pages); + telemetry_notify_vregion_change(); + } +} + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_collect_vpage_stats(struct sof_tel_vregion_section *vsec) +{ + k_spinlock_key_t key; + + if (!vsec) + return; + + key = k_spin_lock(&vpage_tel_lock); + vsec->total_pages = vpage_tel_snap.total_pages; + vsec->free_pages = vpage_tel_snap.free_pages; + vsec->min_free_pages = vpage_tel_snap.min_free_pages; + vsec->num_elems_in_use = vpage_tel_snap.num_elems_in_use; + vsec->alloc_fail_count = vpage_tel_snap.alloc_fail_count; + vsec->region_base_va = vpage_tel_snap.region_base_va; + memcpy(vsec->velems, vpage_tel_snap.velems, sizeof(vsec->velems)); + k_spin_unlock(&vpage_tel_lock, key); } +#endif /** * @brief Initialize virtual page allocator @@ -361,6 +431,7 @@ __cold static int vpage_init(void) vpage_ctx.bitmap.num_bundles = bitmap_num_bundles; vpage_ctx.bitmap.bundles = bundles; vpage_ctx.vpage_blocks.bitmap = &vpage_ctx.bitmap; + vpage_tel_sync_locked(); LOG_INF("region %p size %#zx pages %u", (void *)vpage_ctx.virtual_region->addr, diff --git a/zephyr/lib/vregion.c b/zephyr/lib/vregion.c index 846f6d24356e..d0c3d9ee49f2 100644 --- a/zephyr/lib/vregion.c +++ b/zephyr/lib/vregion.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -117,6 +118,93 @@ struct vregion { static sys_dlist_t vregion_list = SYS_DLIST_STATIC_INIT(&vregion_list); static K_MUTEX_DEFINE(vregion_list_lock); +#ifdef CONFIG_SOF_TELEMETRY_VREGION +static struct k_spinlock tel_vregion_lock; +static struct vregion *tel_vregions[SOF_TEL_MAX_VREGIONS]; +static struct sof_tel_vregion_info tel_vregion_snap[SOF_TEL_MAX_VREGIONS]; +static uint16_t tel_vregions_created; + +static void vregion_tel_fill_info(struct vregion *vr, struct sof_tel_vregion_info *info) +{ + memset(info, 0, sizeof(*info)); + info->base_va = (uint32_t)POINTER_TO_UINT(vr->base); + info->total_pages = (uint16_t)vr->pages; + info->use_count = (uint8_t)MIN(vr->use_count, UINT8_MAX); + info->type_and_flags = (uint8_t)vr->type; + info->lifetime_size_bytes = (uint32_t)vr->lifetime.size; + info->lifetime_used_bytes = (uint32_t)vr->lifetime.used; + info->lifetime_free_count = (uint16_t)MIN(vr->lifetime.free_count, UINT16_MAX); +#if CONFIG_SYS_HEAP_RUNTIME_STATS + if (vr->type == VREGION_MEM_TYPE_INTERIM && vr->interim.heap.heap.heap) { + struct sys_memory_stats stats = { 0 }; + k_spinlock_key_t hkey = k_spin_lock(&vr->interim.heap.lock); + int ret = sys_heap_runtime_stats_get(&vr->interim.heap.heap, &stats); + + k_spin_unlock(&vr->interim.heap.lock, hkey); + if (ret == 0) { + info->interim_alloc_bytes = (uint32_t)stats.allocated_bytes; + info->interim_free_kb = (uint16_t)(stats.free_bytes >> 10); + } + } +#endif +} + +/* Must be called while vr->lock is held (or during creation before vr is published) */ +static void vregion_tel_sync_locked(struct vregion *vr) +{ + struct sof_tel_vregion_info info; + k_spinlock_key_t key; + + vregion_tel_fill_info(vr, &info); + key = k_spin_lock(&tel_vregion_lock); + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (tel_vregions[i] == vr) { + tel_vregion_snap[i] = info; + break; + } + } + k_spin_unlock(&tel_vregion_lock, key); +} + +static void vregion_tel_register(struct vregion *vr) +{ + struct sof_tel_vregion_info info; + k_spinlock_key_t key; + + vregion_tel_fill_info(vr, &info); + key = k_spin_lock(&tel_vregion_lock); + tel_vregions_created++; + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (!tel_vregions[i]) { + tel_vregions[i] = vr; + tel_vregion_snap[i] = info; + break; + } + } + k_spin_unlock(&tel_vregion_lock, key); + telemetry_notify_vregion_change(); +} + +static void vregion_tel_unregister(struct vregion *vr) +{ + k_spinlock_key_t key = k_spin_lock(&tel_vregion_lock); + + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (tel_vregions[i] == vr) { + tel_vregions[i] = NULL; + memset(&tel_vregion_snap[i], 0, sizeof(tel_vregion_snap[i])); + break; + } + } + k_spin_unlock(&tel_vregion_lock, key); + telemetry_notify_vregion_change(); +} +#else +static inline void vregion_tel_sync_locked(struct vregion *vr) { (void)vr; } +static inline void vregion_tel_register(struct vregion *vr) { (void)vr; } +static inline void vregion_tel_unregister(struct vregion *vr) { (void)vr; } +#endif + /** * @brief Create a new virtual region instance. * @@ -197,6 +285,8 @@ __cold struct vregion *vregion_create(size_t memsize) LOG_INF("new at base %p size %#zx pages %u metadata at %p", (void *)vregion_base, total_size, pages, (void *)vr); + vregion_tel_register(vr); + return vr; } @@ -207,6 +297,7 @@ struct vregion *z_impl_vregion_get(struct vregion *vr) k_mutex_lock(&vr->lock, K_FOREVER); vr->use_count++; + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); return vr; @@ -299,6 +390,7 @@ static void vregion_destroy(struct vregion *vr) /* log the vregion being destroyed */ LOG_DBG("destroy %p size %#zx pages %u", (void *)vr->base, vr->size, vr->pages); LOG_DBG(" lifetime used %zu free count %d", vr->lifetime.used, vr->lifetime.free_count); + vregion_tel_unregister(vr); vpage_free(vr->base); vregion_unmap(vr); @@ -319,6 +411,8 @@ static int vregion_put_locked(struct vregion *vr) if (!--vr->use_count) sys_dlist_remove(&vr->node); + else + vregion_tel_sync_locked(vr); return vr->use_count; } @@ -414,6 +508,7 @@ void z_impl_vregion_set_interim(struct vregion *vr) k_mutex_lock(&vr->lock, K_FOREVER); interim_heap_init(vr); + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); } @@ -535,6 +630,7 @@ void z_impl_vregion_free(struct vregion *vr, void *ptr) else LOG_ERR("error: vregion free invalid pointer %p", ptr); + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); } EXPORT_SYMBOL(z_impl_vregion_free); @@ -573,6 +669,7 @@ void *z_impl_vregion_alloc_align(struct vregion *vr, p = NULL; } + vregion_tel_sync_locked(vr); k_mutex_unlock(&vr->lock); #if CONFIG_SYS_HEAP_RUNTIME_STATS @@ -703,8 +800,10 @@ bool vregion_pinned_get(struct vregion *vr) k_mutex_lock(&vr->lock, K_FOREVER); /* do not revive a vregion whose last reference is already gone */ ok = vr->use_count && vr->use_count < UINT_MAX; - if (ok) + if (ok) { vr->use_count++; + vregion_tel_sync_locked(vr); + } k_mutex_unlock(&vr->lock); return ok; @@ -724,3 +823,28 @@ int vregion_pinned_put(struct vregion *vr) return ret; } #endif + +#ifdef CONFIG_SOF_TELEMETRY_VREGION +void telemetry_collect_vregion_stats(struct sof_tel_vregion_section *vsec) +{ + uint16_t active_cnt = 0; + k_spinlock_key_t key; + + if (!vsec) + return; + + key = k_spin_lock(&tel_vregion_lock); + for (int i = 0; i < SOF_TEL_MAX_VREGIONS; i++) { + if (!tel_vregions[i]) { + memset(&vsec->vregions[i], 0, sizeof(vsec->vregions[i])); + continue; + } + active_cnt++; + vsec->vregions[i] = tel_vregion_snap[i]; + } + + vsec->active_vregion_count = active_cnt; + vsec->total_vregions_created = tel_vregions_created; + k_spin_unlock(&tel_vregion_lock, key); +} +#endif