[RNE Rewrite] Add text to speech pipeline implementation

.agents/skills/add-native-extension/SKILL.md

@@ -120,8 +120,8 @@ namespace rnexecutorch::extensions::<domain>
             }
 
             // 6. Perform the computation
-            const float *srcData = reinterpret_cast<const float *>(src->data_.get());
-            float *dstData = reinterpret_cast<float *>(dst->data_.get());
+            const float *srcData = reinterpret_cast<const float *>(src->data_);
+            float *dstData = reinterpret_cast<float *>(dst->data_);
             size_t size = src->size();
 
             for (size_t i = 0; i < size; ++i)

.cspell-wordlist.txt

@@ -275,3 +275,8 @@ pcre
 libkleidicv
 thresholding
 binarization
+subcomponents
+gapless
+Purna
+Viram
+Deergh

apps/speech/app.json

@@ -0,0 +1,45 @@
+{
+  "expo": {
+    "name": "speech",
+    "slug": "speech",
+    "version": "1.0.0",
+    "orientation": "portrait",
+    "icon": "./assets/icons/icon.png",
+    "userInterfaceStyle": "light",
+    "newArchEnabled": true,
+    "scheme": "rne-speech",
+    "splash": {
+      "image": "./assets/icons/splash.png",
+      "resizeMode": "contain",
+      "backgroundColor": "#ffffff"
+    },
+    "ios": {
+      "supportsTablet": true,
+      "bundleIdentifier": "com.anonymous.speech"
+    },
+    "android": {
+      "adaptiveIcon": {
+        "foregroundImage": "./assets/icons/adaptive-icon.png",
+        "backgroundColor": "#ffffff"
+      },
+      "package": "com.anonymous.speech"
+    },
+    "web": {
+      "favicon": "./assets/icons/favicon.png"
+    },
+    "plugins": [
+      "expo-router",
+      [
+        "expo-build-properties",
+        {
+          "android": {
+            "minSdkVersion": 26
+          },
+          "ios": {
+            "deploymentTarget": "17.0"
+          }
+        }
+      ]
+    ]
+  }
+}

apps/speech/app/_layout.tsx

@@ -0,0 +1,32 @@
+import { Drawer } from 'expo-router/drawer';
+import { ColorPalette } from '../theme';
+import React from 'react';
+
+export default function Layout() {
+  return (
+    <Drawer
+      screenOptions={{
+        drawerActiveTintColor: ColorPalette.primary,
+        drawerInactiveTintColor: '#888',
+        headerTintColor: ColorPalette.primary,
+        headerTitleStyle: { color: ColorPalette.primary },
+      }}
+    >
+      <Drawer.Screen
+        name="index"
+        options={{
+          drawerLabel: () => null,
+          title: 'Main Menu',
+          drawerItemStyle: { display: 'none' },
+        }}
+      />
+      <Drawer.Screen
+        name="text-to-speech/index"
+        options={{
+          drawerLabel: 'Text to Speech',
+          title: 'Text to Speech',
+        }}
+      />
+    </Drawer>
+  );
+}

apps/speech/app/index.tsx

@@ -0,0 +1,49 @@
+import { useRouter } from 'expo-router';
+import { View, Text, StyleSheet, TouchableOpacity } from 'react-native';
+import { ColorPalette } from '../theme';
+
+export default function Home() {
+  const router = useRouter();
+
+  return (
+    <View style={styles.container}>
+      <Text style={styles.headerText}>Select a demo model</Text>
+      <View style={styles.buttonContainer}>
+        <TouchableOpacity style={styles.button} onPress={() => router.navigate('text-to-speech/')}>
+          <Text style={styles.buttonText}>Text to Speech</Text>
+        </TouchableOpacity>
+      </View>
+    </View>
+  );
+}
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+    backgroundColor: '#fff',
+  },
+  headerText: {
+    fontSize: 18,
+    color: ColorPalette.strongPrimary,
+    margin: 20,
+  },
+  buttonContainer: {
+    width: '80%',
+    justifyContent: 'space-evenly',
+    marginBottom: 20,
+  },
+  button: {
+    backgroundColor: ColorPalette.strongPrimary,
+    borderRadius: 8,
+    padding: 14,
+    alignItems: 'center',
+    marginBottom: 12,
+  },
+  buttonText: {
+    color: 'white',
+    fontSize: 16,
+    fontWeight: '600',
+  },
+});

apps/speech/app/text-to-speech/index.tsx

@@ -0,0 +1,212 @@
+import React, { useState, useCallback, useEffect, useRef } from 'react';
+import { View, Text, TextInput, StyleSheet, ScrollView, Keyboard } from 'react-native';
+import { commonStyles, theme } from '../../theme';
+import { useTextToSpeech, models } from 'react-native-executorch';
+import ScreenWrapper from '../../components/ScreenWrapper';
+import { VoicePicker, type VoiceOption } from '../../components/VoicePicker';
+import { ModelStatus } from '../../components/ModelStatus';
+import { LatencyIndicator } from '../../components/LatencyIndicator';
+import { Button } from '../../components/Button';
+import type { TextToSpeechModel } from 'react-native-executorch';
+import { AudioManager, AudioContext, AudioBuffer } from 'react-native-audio-api';
+
+const VOICE_OPTIONS: VoiceOption<TextToSpeechModel>[] = [
+  { name: 'AF Heart', lang: 'en-us', value: models.textToSpeech.KOKORO.AF_HEART },
+  { name: 'AF River', lang: 'en-us', value: models.textToSpeech.KOKORO.AF_RIVER },
+  { name: 'AF Sarah', lang: 'en-us', value: models.textToSpeech.KOKORO.AF_SARAH },
+  { name: 'AM Adam', lang: 'en-us', value: models.textToSpeech.KOKORO.AM_ADAM },
+  { name: 'AM Michael', lang: 'en-us', value: models.textToSpeech.KOKORO.AM_MICHAEL },
+  { name: 'BF Emma', lang: 'en-gb', value: models.textToSpeech.KOKORO.BF_EMMA },
+  { name: 'BM Daniel', lang: 'en-gb', value: models.textToSpeech.KOKORO.BM_DANIEL },
+  { name: 'FF Siwis', lang: 'fr', value: models.textToSpeech.KOKORO.FF_SIWIS },
+  { name: 'EF Dora', lang: 'es', value: models.textToSpeech.KOKORO.EF_DORA },
+  { name: 'EM Alex', lang: 'es', value: models.textToSpeech.KOKORO.EM_ALEX },
+  { name: 'IF Sara', lang: 'it', value: models.textToSpeech.KOKORO.IF_SARA },
+  { name: 'IM Nicola', lang: 'it', value: models.textToSpeech.KOKORO.IM_NICOLA },
+  { name: 'PF Dora', lang: 'pt', value: models.textToSpeech.KOKORO.PF_DORA },
+  { name: 'PM Santa', lang: 'pt', value: models.textToSpeech.KOKORO.PM_SANTA },
+  { name: 'HF Alpha', lang: 'hi', value: models.textToSpeech.KOKORO.HF_ALPHA },
+  { name: 'HM Omega', lang: 'hi', value: models.textToSpeech.KOKORO.HM_OMEGA },
+  { name: 'PM Mateusz', lang: 'pl', value: models.textToSpeech.KOKORO.PM_MATEUSZ },
+  { name: 'DF Anna', lang: 'de', value: models.textToSpeech.KOKORO.DF_ANNA },
+];
+
+const createAudioBufferFromVector = (
+  audioVector: Float32Array,
+  audioContext: AudioContext | null = null,
+  sampleRate: number = 24000
+): AudioBuffer => {
+  if (audioContext == null) audioContext = new AudioContext({ sampleRate });
+
+  const audioBuffer = audioContext.createBuffer(1, audioVector.length, sampleRate);
+  const channelData = audioBuffer.getChannelData(0);
+  channelData.set(audioVector);
+
+  return audioBuffer;
+};
+
+function TextToSpeechContent() {
+  const [selectedVoice, setSelectedVoice] = useState<TextToSpeechModel>(VOICE_OPTIONS[0].value);
+  const [text, setText] = useState('');
+  const [isPlaying, setIsPlaying] = useState(false);
+  const [latency, setLatency] = useState<number | null>(null);
+  const [error, setError] = useState<string | null>(null);
+
+  const { isReady, downloadProgress, error: loadError, stream } = useTextToSpeech(selectedVoice);
+
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const gainNodeRef = useRef<any>(null);
+
+  useEffect(() => {
+    AudioManager.setAudioSessionOptions({
+      iosCategory: 'playAndRecord',
+      iosMode: 'spokenAudio',
+      iosOptions: ['defaultToSpeaker'],
+    });
+
+    const context = new AudioContext({ sampleRate: 24000 });
+    audioContextRef.current = context;
+    context.suspend();
+
+    const gainNode = context.createGain();
+    gainNode.gain.value = 2.0;
+    gainNode.connect(context.destination);
+    gainNodeRef.current = gainNode;
+
+    return () => {
+      audioContextRef.current?.close();
+      audioContextRef.current = null;
+      gainNodeRef.current = null;
+    };
+  }, []);
+
+  const handleSpeak = useCallback(async () => {
+    if (!stream || !text.trim()) return;
+
+    Keyboard.dismiss();
+    setIsPlaying(true);
+    setError(null);
+    const start = Date.now();
+
+    const audioContext = audioContextRef.current;
+    if (!audioContext) return;
+
+    if (audioContext.state === 'suspended') {
+      await audioContext.resume();
+    }
+
+    try {
+      await stream({
+        text: text.trim(),
+        speed: 0.9,
+        onBegin: () => {
+          setLatency(null);
+        },
+        // Resolve once this chunk has finished playing so the next one is
+        // delivered only afterwards — giving gapless, sequential playback.
+        onNext: (audioVec: Float32Array) =>
+          new Promise<void>((resolve) => {
+            const audioBuffer = createAudioBufferFromVector(audioVec, audioContext, 24000);
+
+            const source = audioContext.createBufferSource();
+            source.buffer = audioBuffer;
+
+            if (gainNodeRef.current) {
+              source.connect(gainNodeRef.current);
+            } else {
+              source.connect(audioContext.destination);
+            }
+
+            source.onEnded = () => resolve();
+            source.start();
+          }),
+        onEnd: () => {
+          setLatency(Date.now() - start);
+          setIsPlaying(false);
+          audioContext.suspend();
+        },
+      });
+    } catch (e: any) {
+      setError(e.message || String(e));
+      setIsPlaying(false);
+    }
+  }, [stream, text]);
+
+  const activeError = loadError ? String(loadError) : error;
+
+  return (
+    <ScrollView
+      style={commonStyles.container}
+      contentContainerStyle={commonStyles.contentContainer}
+    >
+      <Text style={commonStyles.description}>
+        Enter text and select a voice to generate speech.
+      </Text>
+
+      <VoicePicker
+        label="Voice"
+        options={VOICE_OPTIONS}
+        selectedValue={selectedVoice}
+        disabled={isPlaying}
+        onValueChange={(model) => {
+          setSelectedVoice(model);
+          setError(null);
+          setLatency(null);
+        }}
+      />
+
+      <ModelStatus
+        isReady={isReady}
+        downloadProgress={downloadProgress}
+        error={activeError}
+        modelTypeLabel="TTS model"
+      />
+
+      <TextInput
+        style={styles.textInput}
+        placeholder="Type something to speak..."
+        placeholderTextColor={theme.colors.textPlaceholder}
+        value={text}
+        onChangeText={setText}
+        multiline
+        numberOfLines={4}
+        textAlignVertical="top"
+      />
+
+      <View style={commonStyles.buttonRow}>
+        <Button
+          title={isPlaying ? 'Speaking...' : 'Speak'}
+          onPress={handleSpeak}
+          disabled={!isReady || !text.trim() || isPlaying}
+          loading={isPlaying}
+          variant="primary"
+        />
+      </View>
+
+      <LatencyIndicator latency={latency} />
+    </ScrollView>
+  );
+}
+
+export default function TextToSpeechScreen() {
+  return (
+    <ScreenWrapper>
+      <TextToSpeechContent />
+    </ScreenWrapper>
+  );
+}
+
+const styles = StyleSheet.create({
+  textInput: {
+    width: '100%',
+    minHeight: 100,
+    backgroundColor: theme.colors.cardBackground,
+    borderRadius: theme.radius.medium,
+    borderWidth: 1,
+    borderColor: theme.colors.lightBorder,
+    padding: theme.spacing.medium,
+    fontSize: 15,
+    color: theme.colors.textSecondary,
+    marginBottom: theme.spacing.large,
+  },
+});

apps/speech/babel.config.js

@@ -0,0 +1,7 @@
+module.exports = function (api) {
+  api.cache(true);
+  return {
+    presets: ['babel-preset-expo'],
+    plugins: ['react-native-worklets/plugin'],
+  };
+};