diff options
Diffstat (limited to 'tests/VoiceInteraction/src')
6 files changed, 440 insertions, 27 deletions
diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistProxyActivity.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistProxyActivity.java new file mode 100644 index 0000000..d0c5e36 --- /dev/null +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistProxyActivity.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.test.voiceinteraction; + +import android.app.Activity; +import android.content.Intent; +import android.os.Bundle; + +public class AssistProxyActivity extends Activity { + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + finish(); + Intent intent = new Intent(this, MainInteractionService.class); + intent.setAction(Intent.ACTION_ASSIST); + startService(intent); + } +} diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java new file mode 100644 index 0000000..8a72341 --- /dev/null +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.test.voiceinteraction; + +import android.annotation.Nullable; +import android.app.AssistStructure; +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Paint; +import android.graphics.Rect; +import android.util.AttributeSet; +import android.util.Log; +import android.view.View; + +import java.util.ArrayList; + +public class AssistVisualizer extends View { + static final String TAG = "AssistVisualizer"; + + AssistStructure mAssistStructure; + final Paint mFramePaint = new Paint(); + final ArrayList<Rect> mTextRects = new ArrayList<>(); + final int[] mTmpLocation = new int[2]; + + public AssistVisualizer(Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + setWillNotDraw(false); + mFramePaint.setColor(0xffff0000); + mFramePaint.setStyle(Paint.Style.STROKE); + mFramePaint.setStrokeWidth(0); + } + + public void setAssistStructure(AssistStructure as) { + mAssistStructure = as; + mAssistStructure.dump(); + mTextRects.clear(); + final int N = as.getWindowNodeCount(); + if (N > 0) { + for (int i=0; i<N; i++) { + AssistStructure.WindowNode windowNode = as.getWindowNodeAt(i); + buildTextRects(windowNode.getRootViewNode(), windowNode.getLeft(), + windowNode.getTop()); + } + } + Log.d(TAG, "Building text rects in " + this + ": found " + mTextRects.size()); + invalidate(); + } + + public void clearAssistData() { + mAssistStructure = null; + mTextRects.clear(); + } + + void buildTextRects(AssistStructure.ViewNode root, int parentLeft, int parentTop) { + if (root.getVisibility() != View.VISIBLE) { + return; + } + int left = parentLeft+root.getLeft(); + int top = parentTop+root.getTop(); + if (root.getText() != null || root.getContentDescription() != null) { + Rect r = new Rect(left, top, left+root.getWidth(), top+root.getHeight()); + Log.d(TAG, "View " + root.getClassName() + " " + left + "," + top + " tr " + + r.toShortString() + ": " + + (root.getText() != null ? root.getText() : root.getContentDescription())); + mTextRects.add(r); + } + final int N = root.getChildCount(); + if (N > 0) { + left -= root.getScrollX(); + top -= root.getScrollY(); + for (int i=0; i<N; i++) { + AssistStructure.ViewNode child = root.getChildAt(i); + buildTextRects(child, left, top); + } + } + } + + @Override + protected void onDraw(Canvas canvas) { + super.onDraw(canvas); + getLocationOnScreen(mTmpLocation); + final int N = mTextRects.size(); + Log.d(TAG, "Drawing text rects in " + this + ": found " + mTextRects.size()); + for (int i=0; i<N; i++) { + Rect r = mTextRects.get(i); + canvas.drawRect(r.left-mTmpLocation[0], r.top-mTmpLocation[1], + r.right-mTmpLocation[0], r.bottom-mTmpLocation[1], mFramePaint); + } + } +} diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AsyncStructure.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AsyncStructure.java new file mode 100644 index 0000000..73e04e5 --- /dev/null +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AsyncStructure.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.test.voiceinteraction; + +import android.annotation.Nullable; +import android.content.Context; +import android.util.AttributeSet; +import android.view.View; +import android.view.ViewAssistStructure; +import android.widget.TextView; + +/** + * Test for asynchronously creating additional assist structure. + */ +public class AsyncStructure extends TextView { + public AsyncStructure(Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + } + + @Override + public void onProvideVirtualAssistStructure(ViewAssistStructure structure) { + structure.setChildCount(1); + final ViewAssistStructure child = structure.asyncNewChild(0); + final int width = getWidth(); + final int height = getHeight(); + (new Thread() { + @Override + public void run() { + // Simulate taking a long time to build this. + try { + sleep(2000); + } catch (InterruptedException e) { + } + child.setClassName(AsyncStructure.class.getName()); + child.setVisibility(View.VISIBLE); + child.setDimens(width / 4, height / 4, 0, 0, width / 2, height / 2); + child.setEnabled(true); + child.setContentDescription("This is some async content"); + child.setText("We could have lots and lots of async text!"); + child.asyncCommit(); + } + }).start(); + } +} diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionService.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionService.java index 4639114..15196b4 100644 --- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionService.java +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionService.java @@ -16,6 +16,7 @@ package com.android.test.voiceinteraction; +import android.content.ComponentName; import android.content.Intent; import android.os.Bundle; import android.service.voice.AlwaysOnHotwordDetector; @@ -74,9 +75,14 @@ public class MainInteractionService extends VoiceInteractionService { @Override public int onStartCommand(Intent intent, int flags, int startId) { - Bundle args = new Bundle(); - args.putParcelable("intent", new Intent(this, TestInteractionActivity.class)); - startSession(args); + if (isActiveService(this, new ComponentName(this, getClass()))) { + Bundle args = new Bundle(); + args.putParcelable("intent", new Intent(this, TestInteractionActivity.class)); + args.putBundle("assist", intent.getExtras()); + startSession(args, START_WITH_ASSIST|START_WITH_SCREENSHOT); + } else { + Log.w(TAG, "Not starting -- not current voice interaction service"); + } stopSelf(startId); return START_NOT_STICKY; } diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java index d20906e..ec727c4 100644 --- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/MainInteractionSession.java @@ -16,13 +16,18 @@ package com.android.test.voiceinteraction; +import android.app.AssistContent; +import android.app.AssistStructure; +import android.app.VoiceInteractor; import android.content.Context; import android.content.Intent; +import android.graphics.Bitmap; import android.os.Bundle; import android.service.voice.VoiceInteractionSession; import android.util.Log; import android.view.View; import android.widget.Button; +import android.widget.ImageView; import android.widget.TextView; public class MainInteractionSession extends VoiceInteractionSession @@ -31,20 +36,30 @@ public class MainInteractionSession extends VoiceInteractionSession Intent mStartIntent; View mContentView; + AssistVisualizer mAssistVisualizer; + View mTopContent; + View mBottomContent; TextView mText; Button mStartButton; + ImageView mScreenshot; Button mConfirmButton; Button mCompleteButton; Button mAbortButton; + AssistStructure mAssistStructure; + static final int STATE_IDLE = 0; static final int STATE_LAUNCHING = 1; static final int STATE_CONFIRM = 2; - static final int STATE_COMMAND = 3; - static final int STATE_ABORT_VOICE = 4; - static final int STATE_COMPLETE_VOICE = 5; + static final int STATE_PICK_OPTION = 3; + static final int STATE_COMMAND = 4; + static final int STATE_ABORT_VOICE = 5; + static final int STATE_COMPLETE_VOICE = 6; + static final int STATE_DONE=7; int mState = STATE_IDLE; + VoiceInteractor.PickOptionRequest.Option[] mPendingOptions; + CharSequence mPendingPrompt; Request mPendingRequest; MainInteractionSession(Context context) { @@ -52,31 +67,115 @@ public class MainInteractionSession extends VoiceInteractionSession } @Override - public void onCreate(Bundle args) { + public void onCreate(Bundle args, int startFlags) { super.onCreate(args); - showWindow(); + } + + @Override + public void onShow(Bundle args, int showFlags) { + super.onShow(args, showFlags); + mState = STATE_IDLE; mStartIntent = args.getParcelable("intent"); + if (mAssistVisualizer != null) { + mAssistVisualizer.clearAssistData(); + } + onHandleScreenshot(null); + updateState(); + } + + @Override + public void onHide() { + super.onHide(); + if (mAssistVisualizer != null) { + mAssistVisualizer.clearAssistData(); + } + mState = STATE_DONE; + updateState(); } @Override public View onCreateContentView() { mContentView = getLayoutInflater().inflate(R.layout.voice_interaction_session, null); + mAssistVisualizer = (AssistVisualizer)mContentView.findViewById(R.id.assist_visualizer); + if (mAssistStructure != null) { + mAssistVisualizer.setAssistStructure(mAssistStructure); + } + mTopContent = mContentView.findViewById(R.id.top_content); + mBottomContent = mContentView.findViewById(R.id.bottom_content); mText = (TextView)mContentView.findViewById(R.id.text); mStartButton = (Button)mContentView.findViewById(R.id.start); mStartButton.setOnClickListener(this); + mScreenshot = (ImageView)mContentView.findViewById(R.id.screenshot); mConfirmButton = (Button)mContentView.findViewById(R.id.confirm); mConfirmButton.setOnClickListener(this); mCompleteButton = (Button)mContentView.findViewById(R.id.complete); mCompleteButton.setOnClickListener(this); mAbortButton = (Button)mContentView.findViewById(R.id.abort); mAbortButton.setOnClickListener(this); - updateState(); return mContentView; } + @Override + public void onHandleAssist(Bundle assistBundle) { + if (assistBundle != null) { + parseAssistData(assistBundle); + } else { + Log.i(TAG, "onHandleAssist: NO ASSIST BUNDLE"); + } + } + + @Override + public void onHandleScreenshot(Bitmap screenshot) { + if (screenshot != null) { + mScreenshot.setImageBitmap(screenshot); + mScreenshot.setAdjustViewBounds(true); + mScreenshot.setMaxWidth(screenshot.getWidth()/3); + mScreenshot.setMaxHeight(screenshot.getHeight()/3); + } else { + mScreenshot.setImageDrawable(null); + } + } + + void parseAssistData(Bundle assistBundle) { + if (assistBundle != null) { + Bundle assistContext = assistBundle.getBundle(Intent.EXTRA_ASSIST_CONTEXT); + if (assistContext != null) { + mAssistStructure = AssistStructure.getAssistStructure(assistContext); + if (mAssistStructure != null) { + if (mAssistVisualizer != null) { + mAssistVisualizer.setAssistStructure(mAssistStructure); + } + } + AssistContent content = AssistContent.getAssistContent(assistContext); + if (content != null) { + Log.i(TAG, "Assist intent: " + content.getIntent()); + Log.i(TAG, "Assist clipdata: " + content.getClipData()); + } + return; + } + } + if (mAssistVisualizer != null) { + mAssistVisualizer.clearAssistData(); + } + } + void updateState() { + if (mState == STATE_IDLE) { + mTopContent.setVisibility(View.VISIBLE); + mBottomContent.setVisibility(View.GONE); + mAssistVisualizer.setVisibility(View.VISIBLE); + } else if (mState == STATE_DONE) { + mTopContent.setVisibility(View.GONE); + mBottomContent.setVisibility(View.GONE); + mAssistVisualizer.setVisibility(View.GONE); + } else { + mTopContent.setVisibility(View.GONE); + mBottomContent.setVisibility(View.VISIBLE); + mAssistVisualizer.setVisibility(View.GONE); + } mStartButton.setEnabled(mState == STATE_IDLE); - mConfirmButton.setEnabled(mState == STATE_CONFIRM || mState == STATE_COMMAND); + mConfirmButton.setEnabled(mState == STATE_CONFIRM || mState == STATE_PICK_OPTION + || mState == STATE_COMMAND); mAbortButton.setEnabled(mState == STATE_ABORT_VOICE); mCompleteButton.setEnabled(mState == STATE_COMPLETE_VOICE); } @@ -89,22 +188,48 @@ public class MainInteractionSession extends VoiceInteractionSession } else if (v == mConfirmButton) { if (mState == STATE_CONFIRM) { mPendingRequest.sendConfirmResult(true, null); - } else { + mPendingRequest = null; + mState = STATE_LAUNCHING; + } else if (mState == STATE_PICK_OPTION) { + int numReturn = mPendingOptions.length/2; + if (numReturn <= 0) { + numReturn = 1; + } + VoiceInteractor.PickOptionRequest.Option[] picked + = new VoiceInteractor.PickOptionRequest.Option[numReturn]; + for (int i=0; i<picked.length; i++) { + picked[i] = mPendingOptions[i*2]; + } + mPendingOptions = picked; + if (picked.length <= 1) { + mPendingRequest.sendPickOptionResult(true, picked, null); + mPendingRequest = null; + mState = STATE_LAUNCHING; + } else { + mPendingRequest.sendPickOptionResult(false, picked, null); + updatePickText(); + } + } else if (mPendingRequest != null) { mPendingRequest.sendCommandResult(true, null); + mPendingRequest = null; + mState = STATE_LAUNCHING; } - mPendingRequest = null; - mState = STATE_IDLE; - updateState(); } else if (v == mAbortButton) { mPendingRequest.sendAbortVoiceResult(null); mPendingRequest = null; - mState = STATE_IDLE; - updateState(); } else if (v== mCompleteButton) { mPendingRequest.sendCompleteVoiceResult(null); mPendingRequest = null; - mState = STATE_IDLE; - updateState(); + } + updateState(); + } + + @Override + public void onComputeInsets(Insets outInsets) { + super.onComputeInsets(outInsets); + if (mState != STATE_IDLE) { + outInsets.contentInsets.top = mBottomContent.getTop(); + outInsets.touchableInsets = Insets.TOUCHABLE_INSETS_CONTENT; } } @@ -117,13 +242,40 @@ public class MainInteractionSession extends VoiceInteractionSession public void onConfirm(Caller caller, Request request, CharSequence prompt, Bundle extras) { Log.i(TAG, "onConfirm: prompt=" + prompt + " extras=" + extras); mText.setText(prompt); - mStartButton.setText("Confirm"); + mConfirmButton.setText("Confirm"); mPendingRequest = request; + mPendingPrompt = prompt; mState = STATE_CONFIRM; updateState(); } @Override + public void onPickOption(Caller caller, Request request, CharSequence prompt, + VoiceInteractor.PickOptionRequest.Option[] options, Bundle extras) { + Log.i(TAG, "onPickOption: prompt=" + prompt + " options=" + options + " extras=" + extras); + mConfirmButton.setText("Pick Option"); + mPendingRequest = request; + mPendingPrompt = prompt; + mPendingOptions = options; + mState = STATE_PICK_OPTION; + updatePickText(); + updateState(); + } + + void updatePickText() { + StringBuilder sb = new StringBuilder(); + sb.append(mPendingPrompt); + sb.append(": "); + for (int i=0; i<mPendingOptions.length; i++) { + if (i > 0) { + sb.append(", "); + } + sb.append(mPendingOptions[i].getLabel()); + } + mText.setText(sb.toString()); + } + + @Override public void onCompleteVoice(Caller caller, Request request, CharSequence message, Bundle extras) { Log.i(TAG, "onCompleteVoice: message=" + message + " extras=" + extras); mText.setText(message); @@ -145,7 +297,7 @@ public class MainInteractionSession extends VoiceInteractionSession public void onCommand(Caller caller, Request request, String command, Bundle extras) { Log.i(TAG, "onCommand: command=" + command + " extras=" + extras); mText.setText("Command: " + command); - mStartButton.setText("Finish Command"); + mConfirmButton.setText("Finish Command"); mPendingRequest = request; mState = STATE_COMMAND; updateState(); diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/TestInteractionActivity.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/TestInteractionActivity.java index d64eefa..e195c30 100644 --- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/TestInteractionActivity.java +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/TestInteractionActivity.java @@ -18,19 +18,26 @@ package com.android.test.voiceinteraction; import android.app.Activity; import android.app.VoiceInteractor; +import android.content.ComponentName; import android.os.Bundle; +import android.service.voice.VoiceInteractionService; import android.util.Log; import android.view.Gravity; import android.view.View; import android.view.ViewGroup; import android.widget.Button; +import android.widget.TextView; public class TestInteractionActivity extends Activity implements View.OnClickListener { static final String TAG = "TestInteractionActivity"; VoiceInteractor mInteractor; + VoiceInteractor.Request mCurrentRequest = null; + TextView mLog; Button mAbortButton; Button mCompleteButton; + Button mPickButton; + Button mCancelButton; @Override public void onCreate(Bundle savedInstanceState) { @@ -42,19 +49,26 @@ public class TestInteractionActivity extends Activity implements View.OnClickLis return; } + if (!VoiceInteractionService.isActiveService(this, + new ComponentName(this, MainInteractionService.class))) { + Log.w(TAG, "Not current voice interactor!"); + finish(); + return; + } + setContentView(R.layout.test_interaction); + mLog = (TextView)findViewById(R.id.log); mAbortButton = (Button)findViewById(R.id.abort); mAbortButton.setOnClickListener(this); mCompleteButton = (Button)findViewById(R.id.complete); mCompleteButton.setOnClickListener(this); - - // Framework should take care of these. - getWindow().setGravity(Gravity.TOP); - getWindow().setLayout(ViewGroup.LayoutParams.MATCH_PARENT, - ViewGroup.LayoutParams.WRAP_CONTENT); + mPickButton = (Button)findViewById(R.id.pick); + mPickButton.setOnClickListener(this); + mCancelButton = (Button)findViewById(R.id.cancel); + mCancelButton.setOnClickListener(this); mInteractor = getVoiceInteractor(); - VoiceInteractor.ConfirmationRequest req = new VoiceInteractor.ConfirmationRequest( + mCurrentRequest = new VoiceInteractor.ConfirmationRequest( "This is a confirmation", null) { @Override public void onCancel() { @@ -68,7 +82,7 @@ public class TestInteractionActivity extends Activity implements View.OnClickLis getActivity().finish(); } }; - mInteractor.submitRequest(req); + mInteractor.submitRequest(mCurrentRequest); } @Override @@ -84,11 +98,13 @@ public class TestInteractionActivity extends Activity implements View.OnClickLis @Override public void onCancel() { Log.i(TAG, "Canceled!"); + mLog.append("Canceled abort\n"); } @Override public void onAbortResult(Bundle result) { Log.i(TAG, "Abort result: result=" + result); + mLog.append("Abort: result=" + result + "\n"); getActivity().finish(); } }; @@ -99,15 +115,59 @@ public class TestInteractionActivity extends Activity implements View.OnClickLis @Override public void onCancel() { Log.i(TAG, "Canceled!"); + mLog.append("Canceled complete\n"); } @Override public void onCompleteResult(Bundle result) { Log.i(TAG, "Complete result: result=" + result); + mLog.append("Complete: result=" + result + "\n"); getActivity().finish(); } }; mInteractor.submitRequest(req); + } else if (v == mPickButton) { + VoiceInteractor.PickOptionRequest.Option[] options = + new VoiceInteractor.PickOptionRequest.Option[5]; + options[0] = new VoiceInteractor.PickOptionRequest.Option("One"); + options[1] = new VoiceInteractor.PickOptionRequest.Option("Two"); + options[2] = new VoiceInteractor.PickOptionRequest.Option("Three"); + options[3] = new VoiceInteractor.PickOptionRequest.Option("Four"); + options[4] = new VoiceInteractor.PickOptionRequest.Option("Five"); + VoiceInteractor.PickOptionRequest req = new VoiceInteractor.PickOptionRequest( + "Need to pick something", options, null) { + @Override + public void onCancel() { + Log.i(TAG, "Canceled!"); + mLog.append("Canceled pick\n"); + } + + @Override + public void onPickOptionResult(boolean finished, Option[] selections, Bundle result) { + Log.i(TAG, "Pick result: finished=" + finished + " selections=" + selections + + " result=" + result); + StringBuilder sb = new StringBuilder(); + if (finished) { + sb.append("Pick final result: "); + } else { + sb.append("Pick intermediate result: "); + } + for (int i=0; i<selections.length; i++) { + if (i >= 1) { + sb.append(", "); + } + sb.append(selections[i].getLabel()); + } + mLog.append(sb.toString()); + if (finished) { + getActivity().finish(); + } + } + }; + mInteractor.submitRequest(req); + } else if (v == mCancelButton && mCurrentRequest != null) { + Log.i(TAG, "Cancel request"); + mCurrentRequest.cancel(); } } |