Hook in startRecogniton call

Add required info to the sound model database: users & recognition modes

Change-Id: I6e12cbc6342a2767c0e3d8328c0a3be899ac9952

6 files changed, 207 insertions, 71 deletions

diff --git a/core/java/android/hardware/soundtrigger/Keyphrase.java b/core/java/android/hardware/soundtrigger/Keyphrase.java
index 42fd350..51311bb 100644
--- a/core/java/android/hardware/soundtrigger/Keyphrase.java
+++ b/core/java/android/hardware/soundtrigger/Keyphrase.java
@@ -30,7 +30,11 @@ public class Keyphrase implements Parcelable {
     /** A hint text to display corresponding to this keyphrase, e.g. "Hello There". */
     public final String hintText;
     /** The locale of interest when using this Keyphrase. */
-    public String locale;
+    public final String locale;
+    /** The various recognition modes supported by this keyphrase */
+    public final int recognitionModeFlags;
+    /** The users associated with this keyphrase */
+    public final int[] users;
 
     public static final Parcelable.Creator<Keyphrase> CREATOR
             = new Parcelable.Creator<Keyphrase>() {
@@ -44,13 +48,26 @@ public class Keyphrase implements Parcelable {
     };
 
     private static Keyphrase fromParcel(Parcel in) {
-        return new Keyphrase(in.readInt(), in.readString(), in.readString());
+        int id = in.readInt();
+        String hintText = in.readString();
+        String locale = in.readString();
+        int recognitionModeFlags = in.readInt();
+        int numUsers = in.readInt();
+        int[] users = null;
+        if (numUsers > 0) {
+            users = new int[numUsers];
+            in.readIntArray(users);
+        }
+        return new Keyphrase(id, hintText, locale, recognitionModeFlags, users);
     }
 
-    public Keyphrase(int id, String hintText, String locale) {
+    public Keyphrase(int id, String hintText, String locale, int recognitionModeFlags,
+            int[] users) {
         this.id = id;
         this.hintText = hintText;
         this.locale = locale;
+        this.recognitionModeFlags = recognitionModeFlags;
+        this.users = users;
     }
 
     @Override
@@ -58,6 +75,13 @@ public class Keyphrase implements Parcelable {
         dest.writeInt(id);
         dest.writeString(hintText);
         dest.writeString(locale);
+        dest.writeInt(recognitionModeFlags);
+        if (users != null) {
+            dest.writeInt(users.length);
+            dest.writeIntArray(users);
+        } else {
+            dest.writeInt(0);
+        }
     }
 
     @Override
@@ -98,4 +122,14 @@ public class Keyphrase implements Parcelable {
             return false;
         return true;
     }
+
+    @Override
+    public String toString() {
+        return "Keyphrase[id=" + id + ", text=" + hintText + ", locale=" + locale
+                + ", recognitionModes=" + recognitionModeFlags + "]";
+    }
+
+    protected SoundTrigger.Keyphrase convertToSoundTriggerKeyphrase() {
+        return new SoundTrigger.Keyphrase(id, recognitionModeFlags, locale, hintText, users);
+    }
 }
diff --git a/core/java/android/hardware/soundtrigger/KeyphraseEnrollmentInfo.java b/core/java/android/hardware/soundtrigger/KeyphraseEnrollmentInfo.java
index 2f5de6a..c74134a 100644
--- a/core/java/android/hardware/soundtrigger/KeyphraseEnrollmentInfo.java
+++ b/core/java/android/hardware/soundtrigger/KeyphraseEnrollmentInfo.java
@@ -239,7 +239,8 @@ public class KeyphraseEnrollmentInfo {
      * @param keyphrase The keyphrase that the user needs to be enrolled to.
      * @param locale The locale for which the enrollment needs to be performed.
      *        This is a Java locale, for example "en_US".
-     * @return true, if an enrollment client supports the given keyphrase and the given locale.
+     * @return The metadata, if an enrollment client supports the given keyphrase
+     *         and the given locale, null otherwise.
      */
     public KeyphraseMetadata getKeyphraseMetadata(String keyphrase, String locale) {
         if (mKeyphrases == null || mKeyphrases.length == 0) {
diff --git a/core/java/android/hardware/soundtrigger/KeyphraseSoundModel.java b/core/java/android/hardware/soundtrigger/KeyphraseSoundModel.java
index 4ddba6a..a5ab0d2 100644
--- a/core/java/android/hardware/soundtrigger/KeyphraseSoundModel.java
+++ b/core/java/android/hardware/soundtrigger/KeyphraseSoundModel.java
@@ -65,4 +65,15 @@ public class KeyphraseSoundModel implements Parcelable {
         }
         dest.writeParcelableArray(keyphrases, 0);
     }
+
+    public SoundTrigger.KeyphraseSoundModel convertToSoundTriggerKeyphraseSoundModel() {
+        SoundTrigger.Keyphrase[] stKeyphrases = null;
+        if (keyphrases != null) {
+            stKeyphrases = new SoundTrigger.Keyphrase[keyphrases.length];
+            for (int i = 0; i < keyphrases.length; i++) {
+                stKeyphrases[i] = keyphrases[i].convertToSoundTriggerKeyphrase();
+            }
+        }
+        return new SoundTrigger.KeyphraseSoundModel(uuid, data, stKeyphrases);
+    }
 }
diff --git a/core/java/android/hardware/soundtrigger/SoundTriggerHelper.java b/core/java/android/hardware/soundtrigger/SoundTriggerHelper.java
index 0be068d..3659621 100644
--- a/core/java/android/hardware/soundtrigger/SoundTriggerHelper.java
+++ b/core/java/android/hardware/soundtrigger/SoundTriggerHelper.java
@@ -17,6 +17,7 @@
 package android.hardware.soundtrigger;
 
 import android.hardware.soundtrigger.SoundTrigger.ModuleProperties;
+import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
 import android.hardware.soundtrigger.SoundTrigger.RecognitionEvent;
 import android.util.Slog;
 import android.util.SparseArray;
@@ -56,7 +57,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
     private final ModuleProperties mModuleProperties;
     private final SoundTriggerModule mModule;
 
-    private final SparseArray<Listener> mListeners;
+    private final SparseArray<Listener> mActiveListeners;
 
     private int mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE;
 
@@ -77,7 +78,7 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
     public SoundTriggerHelper() {
         ArrayList <ModuleProperties> modules = new ArrayList<>();
         int status = SoundTrigger.listModules(modules);
-        mListeners = new SparseArray<>(1);
+        mActiveListeners = new SparseArray<>(1);
         if (status != SoundTrigger.STATUS_OK || modules.size() == 0) {
             // TODO: Figure out how to handle errors in listing the modules here.
             dspInfo = null;
@@ -94,27 +95,9 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
     }
 
     /**
-     * @return True, if the given {@link Keyphrase} is supported on DSP.
-     */
-    public boolean isKeyphraseSupported(Keyphrase keyphrase) {
-        // TODO: We also need to look into a SoundTrigger API that let's us
-        // query this. For now just return true.
-        return true;
-    }
-
-    /**
-     * @return True, if the given {@link Keyphrase} has been enrolled.
-     */
-    public boolean isKeyphraseEnrolled(Keyphrase keyphrase) {
-        // TODO: Query VoiceInteractionManagerService
-        // to list registered sound models.
-        return false;
-    }
-
-    /**
      * @return True, if a recognition for the given {@link Keyphrase} is active.
      */
-    public boolean isKeyphraseActive(Keyphrase keyphrase) {
+    public synchronized boolean isKeyphraseActive(Keyphrase keyphrase) {
         // TODO: Check if the recognition for the keyphrase is currently active.
         return false;
     }
@@ -124,55 +107,98 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
      *
      * @param keyphraseId The identifier of the keyphrase for which
      *        the recognition is to be started.
+     * @param soundModel The sound model to use for recognition.
      * @param listener The listener for the recognition events related to the given keyphrase.
      * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
      */
-    public int startRecognition(int keyphraseId, Listener listener) {
+    public synchronized int startRecognition(int keyphraseId,
+            SoundTrigger.KeyphraseSoundModel soundModel,
+            Listener listener, RecognitionConfig recognitionConfig) {
         if (dspInfo == null || mModule == null) {
             Slog.w(TAG, "Attempting startRecognition without the capability");
             return STATUS_ERROR;
         }
 
-        if (mListeners.get(keyphraseId) != listener) {
+        Listener oldListener = mActiveListeners.get(keyphraseId);
+        if (oldListener != null && oldListener != listener) {
             if (mCurrentSoundModelHandle != INVALID_SOUND_MODEL_HANDLE) {
                 Slog.w(TAG, "Canceling previous recognition");
                 // TODO: Inspect the return codes here.
                 mModule.unloadSoundModel(mCurrentSoundModelHandle);
             }
-            mListeners.get(keyphraseId).onListeningStateChanged(STATE_STOPPED);
+            mActiveListeners.get(keyphraseId).onListeningStateChanged(STATE_STOPPED);
+            mActiveListeners.remove(keyphraseId);
+        }
+
+        int[] handle = new int[] { INVALID_SOUND_MODEL_HANDLE };
+        int status = mModule.loadSoundModel(soundModel, handle);
+        if (status != SoundTrigger.STATUS_OK) {
+            Slog.w(TAG, "loadSoundModel call failed with " + status);
+            return STATUS_ERROR;
+        }
+        if (handle[0] == INVALID_SOUND_MODEL_HANDLE) {
+            Slog.w(TAG, "loadSoundModel call returned invalid sound model handle");
+            return STATUS_ERROR;
+        }
+
+        // Start the recognition.
+        status = mModule.startRecognition(handle[0], recognitionConfig);
+        if (status != SoundTrigger.STATUS_OK) {
+            Slog.w(TAG, "startRecognition failed with " + status);
+            return STATUS_ERROR;
         }
 
+        // Everything went well!
+        mCurrentSoundModelHandle = handle[0];
         // Register the new listener. This replaces the old one.
         // There can only be a maximum of one active listener for a keyphrase
         // at any given time.
-        mListeners.put(keyphraseId, listener);
-        // TODO: Get the sound model for the given keyphrase here.
-        // mModule.loadSoundModel(model, soundModelHandle);
-        // mModule.startRecognition(soundModelHandle, data);
-        // mCurrentSoundModelHandle = soundModelHandle;
-        return STATUS_ERROR;
+        mActiveListeners.put(keyphraseId, listener);
+        return STATUS_OK;
     }
 
     /**
      * Stops recognition for the given {@link Keyphrase} if a recognition is currently active.
      *
+     * @param keyphraseId The identifier of the keyphrase for which
+     *        the recognition is to be stopped.
+     * @param listener The listener for the recognition events related to the given keyphrase.
+     *
      * @return One of {@link #STATUS_ERROR} or {@link #STATUS_OK}.
      */
-    public int stopRecognition(int id, Listener listener) {
+    public synchronized int stopRecognition(int keyphraseId, Listener listener) {
         if (dspInfo == null || mModule == null) {
             Slog.w(TAG, "Attempting stopRecognition without the capability");
             return STATUS_ERROR;
         }
 
-        if (mListeners.get(id) != listener) {
+        Listener currentListener = mActiveListeners.get(keyphraseId);
+        if (currentListener == null) {
+            // startRecognition hasn't been called or it failed.
+            Slog.w(TAG, "Attempting stopRecognition without a successful startRecognition");
+            return STATUS_ERROR;
+        } else if (currentListener != listener) {
+            // TODO: Figure out if this should match the listener that was passed in during
+            // startRecognition, or should we allow a different listener to stop the recognition,
+            // in which case we don't need to pass in a listener here.
             Slog.w(TAG, "Attempting stopRecognition for another recognition");
             return STATUS_ERROR;
         } else {
             // Stop recognition if it's the current one, ignore otherwise.
             // TODO: Inspect the return codes here.
-            mModule.stopRecognition(mCurrentSoundModelHandle);
-            mModule.unloadSoundModel(mCurrentSoundModelHandle);
+            int status = mModule.stopRecognition(mCurrentSoundModelHandle);
+            if (status != SoundTrigger.STATUS_OK) {
+                Slog.w(TAG, "stopRecognition call failed with " + status);
+                return STATUS_ERROR;
+            }
+            status = mModule.unloadSoundModel(mCurrentSoundModelHandle);
+            if (status != SoundTrigger.STATUS_OK) {
+                Slog.w(TAG, "unloadSoundModel call failed with " + status);
+                return STATUS_ERROR;
+            }
+
             mCurrentSoundModelHandle = INVALID_SOUND_MODEL_HANDLE;
+            mActiveListeners.remove(keyphraseId);
             return STATUS_OK;
         }
     }
@@ -184,28 +210,26 @@ public class SoundTriggerHelper implements SoundTrigger.StatusListener {
         // TODO: Get the keyphrase out of the event and fire events on it.
         // For now, as a nasty workaround, we fire all events to the listener for
         // keyphrase with TEMP_KEYPHRASE_ID.
+        Listener listener = null;
+        synchronized(this) {
+            // TODO: The keyphrase should come from the recognition event
+            // as it may be for a different keyphrase than the current one.
+            listener = mActiveListeners.get(TEMP_KEYPHRASE_ID);
+        }
+        if (listener == null) {
+            Slog.w(TAG, "received onRecognition event without any listener for it");
+            return;
+        }
 
         switch (event.status) {
             case SoundTrigger.RECOGNITION_STATUS_SUCCESS:
-                // TODO: The keyphrase should come from the recognition event
-                // as it may be for a different keyphrase than the current one.
-                if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
-                    mListeners.get(TEMP_KEYPHRASE_ID).onKeyphraseSpoken();
-                }
+                listener.onKeyphraseSpoken();
                 break;
             case SoundTrigger.RECOGNITION_STATUS_ABORT:
-                // TODO: The keyphrase should come from the recognition event
-                // as it may be for a different keyphrase than the current one.
-                if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
-                    mListeners.get(TEMP_KEYPHRASE_ID).onListeningStateChanged(STATE_STOPPED);
-                }
+                listener.onListeningStateChanged(STATE_STOPPED);
                 break;
             case SoundTrigger.RECOGNITION_STATUS_FAILURE:
-                // TODO: The keyphrase should come from the recognition event
-                // as it may be for a different keyphrase than the current one.
-                if (mListeners.get(TEMP_KEYPHRASE_ID) != null) {
-                    mListeners.get(TEMP_KEYPHRASE_ID).onListeningStateChanged(STATE_STOPPED);
-                }
+                listener.onListeningStateChanged(STATE_STOPPED);
                 break;
         }
     }
diff --git a/core/java/android/service/voice/AlwaysOnHotwordDetector.java b/core/java/android/service/voice/AlwaysOnHotwordDetector.java
index 67ce31e..306543f 100644
--- a/core/java/android/service/voice/AlwaysOnHotwordDetector.java
+++ b/core/java/android/service/voice/AlwaysOnHotwordDetector.java
@@ -20,9 +20,19 @@ import android.content.Intent;
 import android.hardware.soundtrigger.Keyphrase;
 import android.hardware.soundtrigger.KeyphraseEnrollmentInfo;
 import android.hardware.soundtrigger.KeyphraseMetadata;
+import android.hardware.soundtrigger.KeyphraseSoundModel;
+import android.hardware.soundtrigger.SoundTrigger;
+import android.hardware.soundtrigger.SoundTrigger.ConfidenceLevel;
+import android.hardware.soundtrigger.SoundTrigger.KeyphraseRecognitionExtra;
 import android.hardware.soundtrigger.SoundTriggerHelper;
+import android.hardware.soundtrigger.SoundTrigger.RecognitionConfig;
+import android.os.RemoteException;
 import android.util.Slog;
 
+import com.android.internal.app.IVoiceInteractionManagerService;
+
+import java.util.List;
+
 /**
  * A class that lets a VoiceInteractionService implementation interact with
  * always-on keyphrase detection APIs.
@@ -72,11 +82,22 @@ public class AlwaysOnHotwordDetector {
 
     private final String mText;
     private final String mLocale;
-    private final Keyphrase mKeyphrase;
+    /**
+     * The metadata of the Keyphrase, derived from the enrollment application.
+     * This may be null if this keyphrase isn't supported by the enrollment application.
+     */
+    private final KeyphraseMetadata mKeyphraseMetadata;
+    /**
+     * The sound model for the keyphrase, derived from the model management service
+     * (IVoiceInteractionManagerService). May be null if the keyphrase isn't enrolled yet.
+     */
+    private final KeyphraseSoundModel mEnrolledSoundModel;
     private final KeyphraseEnrollmentInfo mKeyphraseEnrollmentInfo;
     private final SoundTriggerHelper mSoundTriggerHelper;
     private final SoundTriggerHelper.Listener mListener;
     private final int mAvailability;
+    private final IVoiceInteractionService mVoiceInteractionService;
+    private final IVoiceInteractionManagerService mModelManagementService;
 
     private int mRecognitionState;
 
@@ -103,25 +124,30 @@ public class AlwaysOnHotwordDetector {
      * @param text The keyphrase text to get the detector for.
      * @param locale The java locale for the detector.
      * @param callback A non-null Callback for receiving the recognition events.
+     * @param voiceInteractionService The current voice interaction service.
+     * @param modelManagementService A service that allows management of sound models.
      *
      * @hide
      */
     public AlwaysOnHotwordDetector(String text, String locale, Callback callback,
             KeyphraseEnrollmentInfo keyphraseEnrollmentInfo,
-            SoundTriggerHelper soundTriggerHelper) {
+            SoundTriggerHelper soundTriggerHelper,
+            IVoiceInteractionService voiceInteractionService,
+            IVoiceInteractionManagerService modelManagementService) {
         mText = text;
         mLocale = locale;
         mKeyphraseEnrollmentInfo = keyphraseEnrollmentInfo;
-        KeyphraseMetadata keyphraseMetadata =
-                mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
-        if (keyphraseMetadata != null) {
-            mKeyphrase = new Keyphrase(keyphraseMetadata.id, text, locale);
-        } else {
-            mKeyphrase = null;
-        }
+        mKeyphraseMetadata = mKeyphraseEnrollmentInfo.getKeyphraseMetadata(text, locale);
         mListener = new SoundTriggerListener(callback);
         mSoundTriggerHelper = soundTriggerHelper;
-        mAvailability = getAvailabilityInternal();
+        mVoiceInteractionService = voiceInteractionService;
+        mModelManagementService = modelManagementService;
+        if (mKeyphraseMetadata != null) {
+            mEnrolledSoundModel = internalGetKeyphraseSoundModel(mKeyphraseMetadata.id);
+        } else {
+            mEnrolledSoundModel = null;
+        }
+        mAvailability = internalGetAvailability();
     }
 
     /**
@@ -171,7 +197,16 @@ public class AlwaysOnHotwordDetector {
         }
 
         mRecognitionState = RECOGNITION_REQUESTED;
-        int code = mSoundTriggerHelper.startRecognition(mKeyphrase.id, mListener);
+        mRecognitionState = RECOGNITION_REQUESTED;
+        KeyphraseRecognitionExtra[] recognitionExtra = new KeyphraseRecognitionExtra[1];
+        // TODO: Do we need to do something about the confidence level here?
+        // TODO: Read the recognition mode flag from the KeyphraseMetadata.
+        // TODO: Take in captureTriggerAudio as a method param here.
+        recognitionExtra[0] = new KeyphraseRecognitionExtra(mKeyphraseMetadata.id,
+                SoundTrigger.RECOGNITION_MODE_VOICE_TRIGGER, new ConfidenceLevel[0]);
+        int code = mSoundTriggerHelper.startRecognition(mKeyphraseMetadata.id,
+                mEnrolledSoundModel.convertToSoundTriggerKeyphraseSoundModel(), mListener,
+                new RecognitionConfig(false, recognitionExtra, null /* additional data */));
         if (code != SoundTriggerHelper.STATUS_OK) {
             Slog.w(TAG, "startRecognition() failed with error code " + code);
             return STATUS_ERROR;
@@ -195,7 +230,8 @@ public class AlwaysOnHotwordDetector {
         }
 
         mRecognitionState = RECOGNITION_NOT_REQUESTED;
-        int code = mSoundTriggerHelper.stopRecognition(mKeyphrase.id, mListener);
+        int code = mSoundTriggerHelper.stopRecognition(mKeyphraseMetadata.id, mListener);
+
         if (code != SoundTriggerHelper.STATUS_OK) {
             Slog.w(TAG, "stopRecognition() failed with error code " + code);
             return STATUS_ERROR;
@@ -230,19 +266,51 @@ public class AlwaysOnHotwordDetector {
         return mKeyphraseEnrollmentInfo.getManageKeyphraseIntent(action, mText, mLocale);
     }
 
-    private int getAvailabilityInternal() {
+    private int internalGetAvailability() {
+        // No DSP available
         if (mSoundTriggerHelper.dspInfo == null) {
             return KEYPHRASE_HARDWARE_UNAVAILABLE;
         }
-        if (mKeyphrase == null || !mSoundTriggerHelper.isKeyphraseSupported(mKeyphrase)) {
+        // No enrollment application supports this keyphrase/locale
+        if (mKeyphraseMetadata == null) {
             return KEYPHRASE_UNSUPPORTED;
         }
-        if (!mSoundTriggerHelper.isKeyphraseEnrolled(mKeyphrase)) {
+        // This keyphrase hasn't been enrolled.
+        if (mEnrolledSoundModel == null) {
             return KEYPHRASE_UNENROLLED;
         }
         return KEYPHRASE_ENROLLED;
     }
 
+    /**
+     * @return The corresponding {@link KeyphraseSoundModel} or null if none is found.
+     */
+    private KeyphraseSoundModel internalGetKeyphraseSoundModel(int keyphraseId) {
+        List<KeyphraseSoundModel> soundModels;
+        try {
+            soundModels = mModelManagementService
+                    .listRegisteredKeyphraseSoundModels(mVoiceInteractionService);
+            if (soundModels == null || soundModels.isEmpty()) {
+                Slog.i(TAG, "No available sound models for keyphrase ID: " + keyphraseId);
+                return null;
+            }
+            for (KeyphraseSoundModel soundModel : soundModels) {
+                if (soundModel.keyphrases == null) {
+                    continue;
+                }
+                for (Keyphrase keyphrase : soundModel.keyphrases) {
+                    // TODO: Check the user handle here to only load a model for the current user.
+                    if (keyphrase.id == keyphraseId) {
+                        return soundModel;
+                    }
+                }
+            }
+        } catch (RemoteException e) {
+            Slog.w(TAG, "RemoteException in listRegisteredKeyphraseSoundModels!");
+        }
+        return null;
+    }
+
     /** @hide */
     static final class SoundTriggerListener implements SoundTriggerHelper.Listener {
         private final Callback mCallback;
diff --git a/core/java/android/service/voice/VoiceInteractionService.java b/core/java/android/service/voice/VoiceInteractionService.java
index cf8d502..a9b1959 100644
--- a/core/java/android/service/voice/VoiceInteractionService.java
+++ b/core/java/android/service/voice/VoiceInteractionService.java
@@ -104,10 +104,8 @@ public class VoiceInteractionService extends Service {
      */
     public final AlwaysOnHotwordDetector getAlwaysOnHotwordDetector(
             String keyphrase, String locale, AlwaysOnHotwordDetector.Callback callback) {
-        // TODO: Cache instances and return the same one instead of creating a new interactor
-        // for the same keyphrase/locale combination.
         return new AlwaysOnHotwordDetector(keyphrase, locale, callback,
-                mKeyphraseEnrollmentInfo, mSoundTriggerHelper);
+                mKeyphraseEnrollmentInfo, mSoundTriggerHelper, mInterface, mSystemService);
     }
 
     /**