audio: tflm: soft mel-log AGC for level-robust wake-word detection

Add a soft AGC in the Q9.23 mel-log domain that automatically clamps
peak energy to prevent clipping on loud utterances, while allowing
recovery toward a 0 dB target at 0.5 dB/sec during active speech.
Requantization is mapped symmetrically from [-1.0, +1.0] in Q9.23 to
[-128, 127] in int8 to match model calibration.

In addition, track a 49-hop VAD history bitmask from the MFCC header.
When the entire sliding context window contains only silence (VAD=0),
int8 requantization and neural network inference execution are gated
off, substantially reducing DSP MCPS load during idle periods.

Signed-off-by: Seppo Ingalsuo <seppo.ingalsuo@linux.intel.com>

Author: singalsu

src/audio/tensorflow/speech.cc

@@ -18,7 +18,6 @@
 
 // The following values are derived from values used during model training.
 // If you change the way you preprocess the input, update all these constants.
-//constexpr int kAudioSampleFrequency = TFLM_SAMPLE_RATE;
 static constexpr int kFeatureSize = TFLM_FEATURE_SIZE;
 static constexpr int kFeatureCount = TFLM_FEATURE_COUNT;
 static constexpr int kFeatureElementCount = TFLM_FEATURE_ELEM_COUNT;
@@ -56,32 +55,6 @@ int RegisterOps(MicroSpeechOpResolver *op_resolver) {
 
 static int Init_Interpreter(struct tf_classify *tfc);
 
-// Decompose the Q9.23 -> int8 requantize factor
-//     M = 1 / (input_scale * 2^23)
-// into a normalized int32 multiplier in [2^30, 2^31) and a right-shift, so the
-// runtime hot path can do (mel_q23 * mult) >> shift in pure integer math.
-// Unpacks IEEE 754 bits directly rather than calling frexpf so that this file
-// pulls in no libm symbols on the minimal-libc SOF build.
-static void ComputeInputRequantizeMultiplier(float input_scale,
-					     int32_t *out_mult, int *out_shift)
-{
-	float m = 1.0f / (input_scale * static_cast<float>(1 << 23));
-	if (!(m > 0.0f)) {
-		*out_mult = 0;
-		*out_shift = 0;
-		return;
-	}
-	uint32_t bits;
-	std::memcpy(&bits, &m, sizeof(bits));
-	// IEEE 754 binary32: bias-127 exponent; frexp uses [0.5, 1.0) => bias-126.
-	int exp = static_cast<int>((bits >> 23) & 0xffu) - 126;
-	// Reinsert the implicit leading 1, then shift the 24-bit mantissa into bit
-	// 30 so the result is Q0.31 with MSB set, i.e. in [2^30, 2^31).
-	int32_t mult = static_cast<int32_t>(((bits & 0x7fffffu) | 0x800000u) << 7);
-	*out_mult = mult;
-	*out_shift = 31 - exp;
-}
-
 int TF_InitOps(struct tf_classify *tfc)
 {
 	op_resolver = new MicroSpeechOpResolver();
@@ -160,14 +133,6 @@ static int Init_Interpreter(struct tf_classify *tfc)
 		return -EINVAL;
 	}
 
-	// expose the model's real input quantization params so callers can
-	// requantize their features correctly instead of assuming a fixed
-	// scale/zero_point.
-	tfc->input_scale = input->params.scale;
-	tfc->input_zero_point = input->params.zero_point;
-	ComputeInputRequantizeMultiplier(tfc->input_scale,
-					 &tfc->input_mult, &tfc->input_shift);
-
 	return 0;
 }
 

src/audio/tensorflow/speech.h

@@ -31,15 +31,6 @@ struct tf_classify {
 	int op_count;
 	uint32_t node_cycles[10];
 	int node_codes[10];
-	float input_scale;
-	int input_zero_point;
-	/* Precomputed integer requantizer for the Q9.23 mel-log -> int8
-	 * feature path: pre_zp = round((int64)mel_q23 * input_mult >> input_shift).
-	 * The factor of 2^-23 from Q9.23 is folded into input_mult so runtime
-	 * does no float math. See Init_Interpreter() in speech.cc.
-	 */
-	int32_t input_mult;
-	int input_shift;
 };
 
 /* Export of C++ APIs into C namespace for linkage */

src/audio/tensorflow/tflm-classify.c

@@ -109,6 +109,29 @@ static const char * const prediction[] = TFLM_CATEGORY_DATA;
 #define TFLM_MFCC_HOP_MS            20
 #define TFLM_INFERENCE_STRIDE_HOPS  25
 
+/* Soft mel-log AGC (units: Q9.23, matches MFCC output). One decade = +10 dB.
+ * Target 0 dB, floor -20 dB. Attack: instant clamp so peak+gain never exceeds
+ * MEL_CLIP_MAX_Q23 (+1.0, +10 dB). Release: additive step per hop during active
+ * speech (vad_flag == 1), chosen so recovery is ~0.5 dB/sec (hop=20 ms, 50 hops/sec).
+ */
+#define TFLM_AGC_GAIN_TARGET_Q23    0 /* 0 dB */
+#define TFLM_AGC_GAIN_FLOOR_Q23     -16777216 /* -20 dB, int32(-20 * 0.1 * 2^23) */
+#define TFLM_AGC_RELEASE_STEP_Q23   8389 /* 0.5 dB/s, int32((0.5 * 0.1 / 50) * 2^23) */
+
+/* This needs to match with sof_tflm_train.py and sof_tflm_verify.py.
+ *
+ * The range -1.0 to +1.0 of Q9.23 Mel values is scaled to +/-1.0 Q1.7.
+ *
+ * in = [-1.0 1.0]; out = [-1 1];
+ * offset = -(in(1) + in(2)) / 2 = 0
+ * scale = (out(2) - out(1)) / (in(2) - in(1)) = 1.0
+ */
+#define MEL_OFFSET_Q23		0 /* 0 */
+#define MEL_SCALE_Q30		(1 << 30) /* 1.0 in Q30 */
+#define MEL_CLIP_MAX_Q23	(1 << 23) /* +1.0 in Q23 */
+#define MEL_CLIP_MAX_Q7		127
+#define MEL_CLIP_MIN_Q7		-128
+
 struct tflm_comp_data {
 	struct comp_data_blob_handler *model_handler;
 	struct tf_classify tfc;
@@ -122,6 +145,10 @@ struct tflm_comp_data {
 	/* Per-instance sliding window and inference cadence. */
 	int8_t feature_buf[TFLM_FEATURE_ELEM_COUNT];
 	int frame_counter;
+	/* Bitmask of VAD flags for the TFLM_FEATURE_COUNT frames in feature_buf. */
+	uint64_t vad_history;
+	/* Persistent AGC gain applied to every Q9.23 mel value (see AGC defines). */
+	int32_t agc_gain_q23;
 	/* Per-instance shutdown-summary counters. */
 	uint32_t category_totals[TFLM_CATEGORY_COUNT];
 	uint32_t total_inferences;
@@ -190,15 +217,11 @@ static __maybe_unused void tflm_send_keyword_notification(struct processing_modu
 
 /* Shared TFLM backend state. speech.cc has one static arena/model/interpreter,
  * so TF_SetModel/TF_InitOps must run only once no matter how many tflmcly
- * instances the topology creates; secondary instances copy the resolved input
- * quant params from the first-init cache below.
+ * instances the topology creates; secondary instances attach to the first-init
+ * instance.
  */
 static bool g_tflm_initialized;
 static int g_tflm_instance_count;
-static float g_tflm_shared_input_scale;
-static int g_tflm_shared_input_zero_point;
-static int32_t g_tflm_shared_input_mult;
-static int32_t g_tflm_shared_input_shift;
 static int g_tflm_shared_categories;
 
 __cold static void tflm_log_summary_at_shutdown(struct processing_module *mod)
@@ -251,6 +274,7 @@ __cold static int tflm_init(struct processing_module *mod)
 	md->private = cd;
 	cd->tfc.categories = TFLM_CATEGORY_COUNT;
 	cd->drain_req_ms = TFLM_KPB_DRAIN_REQ_MS;
+	cd->agc_gain_q23 = TFLM_AGC_GAIN_TARGET_Q23;
 #if CONFIG_AMS
 	cd->kpd_uuid_id = AMS_INVALID_MSG_TYPE;
 #endif
@@ -283,39 +307,6 @@ __cold static int tflm_set_config(struct processing_module *mod, uint32_t param_
 	return 0;
 }
 
-/*
- * MFCC's mel-log output is Q9.23 fixed point, normalized by the mel40.conf
- * profile's mel_offset/mel_scale/top_db tuning to approximately the 0..1
- * range (see the dynamic_mmax clamp + offset + scale in mfcc_common.c).
- * We requantize that into int8 features using the model's own input tensor
- * scale/zero_point (populated by TF_InitOps()/Init_Interpreter() in speech.cc).
- *
- * The float-domain operation is
- *   int8_pre = round((mel_q23 / 2^23) / input_scale) + input_zero_point
- * which we execute in integer math using a pre-decomposed multiplier and
- * right-shift so this per-sample hot path (40 features x ~50 hops/sec)
- * needs no float divides or FPU support:
- *   int8_pre = round((int64)mel_q23 * input_mult >> input_shift) + input_zero_point
- * with input_mult/input_shift set once at model prepare time.
- */
-static inline int8_t mfcc_mel_q23_to_int8(int32_t mel_q23, int32_t input_mult,
-					   int input_shift, int input_zero_point)
-{
-	int64_t prod = (int64_t)mel_q23 * (int64_t)input_mult;
-	int64_t abs_prod = prod >= 0 ? prod : -prod;
-	int64_t rounding = (int64_t)1 << (input_shift - 1);
-	int32_t abs_scaled = (int32_t)((abs_prod + rounding) >> input_shift);
-	int32_t scaled = prod >= 0 ? abs_scaled : -abs_scaled;
-
-	scaled += input_zero_point;
-	if (scaled > 127)
-		scaled = 127;
-	else if (scaled < -128)
-		scaled = -128;
-
-	return (int8_t)scaled;
-}
-
 /*
  * This expects features from 16kHz mono 16 bit input stream.
  *
@@ -332,9 +323,6 @@ static inline int8_t mfcc_mel_q23_to_int8(int32_t mel_q23, int32_t input_mult,
  * features in the input buffer.
  */
 
-
-
-
 static int tflm_process(struct processing_module *mod,
 			struct sof_source **sources, int num_of_sources,
 			struct sof_sink **sinks, int num_of_sinks)
@@ -396,45 +384,61 @@ static int tflm_process(struct processing_module *mod,
 			 * TFLM_FEATURE_SIZE int32 Q9.23 mel-log values into
 			 * int8 features for this hop.
 			 */
+			const struct mfcc_data_header *hdr =
+				(const struct mfcc_data_header *)hop_scratch;
 			const int32_t *mel = (const int32_t *)
 				(hop_scratch + sizeof(struct mfcc_data_header));
 			int8_t hop_features[TFLM_FEATURE_SIZE];
 
-			/* Match training preprocessing exactly: clip Q9.23 mel to
-			 * [-1.0, +4.0] and then rescale to [-1.0, +1.0] centered
-			 * on 0 (np.clip + (X - 1.5) / 2.5 in train_wov_tflm.py).
-			 * TFLite calibration on that range picks
-			 * input_scale=1/128, input_zero_point=0, so the model
-			 * consumes the full int8 dynamic range.
-			 */
-			enum {
-				MEL_CLIP_MIN_Q23 = -(1 << 23),        /* -1.0 */
-				MEL_CLIP_MAX_Q23 =  (4 << 23),        /* +4.0 */
-				MEL_CENTER_Q23   =  (3 << 22),        /* +1.5 */
-				TWO_FIFTHS_Q31   =  858993459,        /* round(0.4 * (1<<31)) */
-			};
-
-			for (int i = 0; i < TFLM_FEATURE_SIZE; i++) {
-				int32_t mel_c = mel[i];
-
-				if (mel_c < MEL_CLIP_MIN_Q23)
-					mel_c = MEL_CLIP_MIN_Q23;
-				else if (mel_c > MEL_CLIP_MAX_Q23)
-					mel_c = MEL_CLIP_MAX_Q23;
-
-				/* Fold (mel - 1.5) * (2/5) into one Q1.31 multiply. */
-				mel_c = Q_MULTSR_32X32((int64_t)(mel_c - MEL_CENTER_Q23),
-						       TWO_FIFTHS_Q31, 23, 31, 23);
-
-				hop_features[i] = mfcc_mel_q23_to_int8(mel_c,
-					cd->tfc.input_mult, cd->tfc.input_shift,
-					cd->tfc.input_zero_point);
+			/* Update VAD history bitmask (tracks last TFLM_FEATURE_COUNT hops). */
+			cd->vad_history = ((cd->vad_history << 1) | (hdr->vad_flag ? 1ULL : 0ULL)) &
+					  ((1ULL << TFLM_FEATURE_COUNT) - 1);
+
+			/* AGC: attack on this hop's peak (always track energy). */
+			int32_t hop_peak_q23 = mel[0];
+			for (int i = 1; i < TFLM_FEATURE_SIZE; i++)
+				if (mel[i] > hop_peak_q23)
+					hop_peak_q23 = mel[i];
+
+			int32_t clip_headroom_q23 = MEL_CLIP_MAX_Q23 - hop_peak_q23;
+			if (cd->agc_gain_q23 > clip_headroom_q23)
+				cd->agc_gain_q23 = clip_headroom_q23;
+			if (cd->agc_gain_q23 < TFLM_AGC_GAIN_FLOOR_Q23)
+				cd->agc_gain_q23 = TFLM_AGC_GAIN_FLOOR_Q23;
+
+			int32_t agc_gain_q23 = cd->agc_gain_q23;
+
+			comp_info(mod->dev, "tflm agc: peak_q23=%d gain_q23=%d",
+				  hop_peak_q23, agc_gain_q23);
+
+			/* Requantize to int8: skip if no speech in the 49-hop window (fill with silence) */
+			if (cd->vad_history) {
+				for (int i = 0; i < TFLM_FEATURE_SIZE; i++) {
+					int32_t mel_c = mel[i] + agc_gain_q23;
+
+					/* Rescale asymmetrically with offset and gain. */
+					mel_c = Q_MULTSR_32X32((int64_t)(mel_c + MEL_OFFSET_Q23),
+							       MEL_SCALE_Q30, 23, 30, 7);
+					if (mel_c > MEL_CLIP_MAX_Q7)
+						mel_c = MEL_CLIP_MAX_Q7;
+					else if (mel_c < MEL_CLIP_MIN_Q7)
+						mel_c = MEL_CLIP_MIN_Q7;
+
+					hop_features[i] = (int8_t)mel_c;
+				}
+			} else {
+				memset(hop_features, MEL_CLIP_MIN_Q7, sizeof(hop_features));
+			}
+
+			/* Release: creep back toward 0 dB target. */
+			if (cd->agc_gain_q23 < TFLM_AGC_GAIN_TARGET_Q23) {
+				cd->agc_gain_q23 += TFLM_AGC_RELEASE_STEP_Q23;
+				if (cd->agc_gain_q23 > TFLM_AGC_GAIN_TARGET_Q23)
+					cd->agc_gain_q23 = TFLM_AGC_GAIN_TARGET_Q23;
 			}
 
 #if CONFIG_COMP_TENSORFLOW_DEBUG_TRACE
 			{
-				const struct mfcc_data_header *hdr =
-					(const struct mfcc_data_header *)hop_scratch;
 				static int dbg_hop_count;
 				int32_t mel_min = mel[0], mel_max = mel[0];
 				int8_t f_min = hop_features[0], f_max = hop_features[0];
@@ -448,11 +452,12 @@ static int tflm_process(struct processing_module *mod,
 					if (hop_features[i] > f_max) f_max = hop_features[i];
 				}
 				snprintk(dbg_buf, sizeof(dbg_buf),
-					 "[DBG hop %d] vad=%d E=%d Ne=%d mel_min=%d mel_max=%d f_min=%d f_max=%d",
+					 "[DBG hop %d] vad=%d E=%d Ne=%d mel_min=%d mel_max=%d f_min=%d f_max=%d agc_q23=%d",
 					 dbg_hop_count, (int)hdr->vad_flag,
 					 (int)hdr->energy, (int)hdr->noise_energy,
 					 mel_min, mel_max,
-					 f_min, f_max);
+					 f_min, f_max,
+					 (int)cd->agc_gain_q23);
 				sof_ut_log(dbg_buf);
 			}
 #endif
@@ -506,6 +511,14 @@ static int tflm_process(struct processing_module *mod,
 			if (cd->frame_counter >= TFLM_INFERENCE_STRIDE_HOPS) {
 				cd->frame_counter = 0;
 
+				/* VAD gate: skip inference computation if no active speech in the 49-hop window */
+				if (!cd->vad_history) {
+#if CONFIG_COMP_TENSORFLOW_DEBUG_TRACE
+					sof_ut_log("[DBG inference] skipped: VAD=0 in entire 49-hop window");
+#endif
+					continue;
+				}
+
 				cd->tfc.audio_features = feature_buf;
 				cd->tfc.audio_data_size = TFLM_FEATURE_ELEM_COUNT;
 
@@ -605,8 +618,9 @@ static int tflm_process(struct processing_module *mod,
 					}
 #endif
 
-					// Only announce a keyword hit for real keyword classes
-					// (indices >= 2, i.e. not silence/unknown).
+					/* Only announce a keyword hit for real keyword classes
+					 * (indices >= 2, i.e. not silence/unknown).
+					 */
 					if (max_idx >= 2 && max_score >= 0.50f) {
 						char kw_buf[96];
 						int max_pct = (int)(max_score * 100.0f);
@@ -651,14 +665,7 @@ static int tflm_prepare(struct processing_module *mod,
 	printk("[TFLM PREPARE] tflm_prepare called, loading model...\n");
 
 	if (g_tflm_initialized) {
-		/* Shared TFLM engine already up. Copy the resolved input quant
-		 * params so this instance's requantize path uses the same
-		 * scale/zero_point/mult/shift as the first-initialized instance.
-		 */
-		cd->tfc.input_scale      = g_tflm_shared_input_scale;
-		cd->tfc.input_zero_point = g_tflm_shared_input_zero_point;
-		cd->tfc.input_mult       = g_tflm_shared_input_mult;
-		cd->tfc.input_shift      = g_tflm_shared_input_shift;
+		/* Shared TFLM engine already up. */
 		cd->tfc.categories       = g_tflm_shared_categories;
 		printk("[TFLM PREPARE] shared engine already initialized; attach instance\n");
 		goto post_init;
@@ -677,17 +684,11 @@ static int tflm_prepare(struct processing_module *mod,
 		return ret;
 	}
 
-	g_tflm_shared_input_scale      = cd->tfc.input_scale;
-	g_tflm_shared_input_zero_point = cd->tfc.input_zero_point;
-	g_tflm_shared_input_mult       = cd->tfc.input_mult;
-	g_tflm_shared_input_shift      = cd->tfc.input_shift;
 	g_tflm_shared_categories       = cd->tfc.categories;
 	g_tflm_initialized = true;
 	printk("[TFLM PREPARE] TFLM model & ops initialized successfully!\n");
 	printk("[TFLM PREPARE] arena_used=%zu / capacity=%zu bytes\n",
 	       TF_ArenaUsedBytes(), TF_ArenaCapacity());
-	printk("[DBG quant] input_scale_x1e6=%d input_zero_point=%d\n",
-	       (int)(cd->tfc.input_scale * 1000000.0f), cd->tfc.input_zero_point);
 
 post_init:
 #if CONFIG_AMS
@@ -711,6 +712,9 @@ static int tflm_reset(struct processing_module *mod)
 	struct tflm_comp_data *cd = module_get_private_data(mod);
 
 	tflm_log_summary_at_shutdown(mod);
+	cd->vad_history = 0;
+	cd->frame_counter = 0;
+	memset(cd->feature_buf, 0, sizeof(cd->feature_buf));
 #if CONFIG_AMS
 	if (cd->kpd_uuid_id != AMS_INVALID_MSG_TYPE) {
 		int ret = ams_helper_unregister_producer(mod->dev,
@@ -730,7 +734,6 @@ static const struct module_interface tflmcly_interface = {
 	.prepare = tflm_prepare,
 	.process = tflm_process,
 	.set_configuration = tflm_set_config,
-//	.get_configuration = tflm_get_config,
 	.reset = tflm_reset,
 	.free = tflm_free
 };