From 6f0fdc4167ad906fc7409856e86cd0cc601c3b53 Mon Sep 17 00:00:00 2001 From: Dianne Hackborn Date: Tue, 7 Jul 2015 14:29:36 -0700 Subject: Implement better handling of text in assist. TextView is now much smarter about the text it reports, limiting it to what is visible (plus a bit more). Also add a facility for it to report where the lines of text are, both as offsets in the text string and their baselines on screen. Part of fixing issue #22328792: Fix scalability issues in AssistStructure Change-Id: Idddb8c3a3331355f381e2d4af06d520fe7c7ce8e --- tests/VoiceInteraction/res/layout/main.xml | 9 +- tests/VoiceInteraction/res/values/strings.xml | 229 +++++++++++++++++++++ .../test/voiceinteraction/AssistVisualizer.java | 26 ++- 3 files changed, 262 insertions(+), 2 deletions(-) (limited to 'tests') diff --git a/tests/VoiceInteraction/res/layout/main.xml b/tests/VoiceInteraction/res/layout/main.xml index 092d37d..0f968eb 100644 --- a/tests/VoiceInteraction/res/layout/main.xml +++ b/tests/VoiceInteraction/res/layout/main.xml @@ -30,10 +30,17 @@ android:layout_width="match_parent" android:layout_height="wrap_content" android:paddingTop="64dp" - android:paddingBottom="64dp" android:text="@string/asyncStructure" /> + + diff --git a/tests/VoiceInteraction/res/values/strings.xml b/tests/VoiceInteraction/res/values/strings.xml index cf660e6..ab39f99 100644 --- a/tests/VoiceInteraction/res/values/strings.xml +++ b/tests/VoiceInteraction/res/values/strings.xml @@ -30,4 +30,233 @@ Cancel Jump out + This is a bunch of text that we will use to show how we handle it +when reporting it for assist data. We need many many lines of text, like\n +this\n +and\n +this other\n +one\n +two\n +three\n +four\n +five\n +six\n +seven\n +eight\n +nine\n +ten\n +eleven\n +twelve\n +thirteen\n +fourteen\n +fifteen\n +sixteen\n +seventeen\n +eighteen\n +nineteen\n +twenty\n +So shaken as we are, so wan with care,\n +Find we a time for frighted peace to pant,\n +And breathe short-winded accents of new broils\n +To be commenced in strands afar remote.\n +No more the thirsty entrance of this soil\n +Shall daub her lips with her own children\'s blood;\n +Nor more shall trenching war channel her fields,\n +Nor bruise her flowerets with the armed hoofs\n +Of hostile paces: those opposed eyes,\n +Which, like the meteors of a troubled heaven,\n +All of one nature, of one substance bred,\n +Did lately meet in the intestine shock\n +And furious close of civil butchery\n +Shall now, in mutual well-beseeming ranks,\n +March all one way and be no more opposed\n +Against acquaintance, kindred and allies:\n +The edge of war, like an ill-sheathed knife,\n +No more shall cut his master. Therefore, friends,\n +As far as to the sepulchre of Christ,\n +Whose soldier now, under whose blessed cross\n +We are impressed and engaged to fight,\n +Forthwith a power of English shall we levy;\n +Whose arms were moulded in their mothers\' womb\n +To chase these pagans in those holy fields\n +Over whose acres walk\'d those blessed feet\n +Which fourteen hundred years ago were nail\'d\n +For our advantage on the bitter cross.\n +But this our purpose now is twelve month old,\n +And bootless \'tis to tell you we will go:\n +Therefore we meet not now. Then let me hear\n +Of you, my gentle cousin Westmoreland,\n +What yesternight our council did decree\n +In forwarding this dear expedience.\n +\n +Hear him but reason in divinity,\n +And all-admiring with an inward wish\n +You would desire the king were made a prelate:\n +Hear him debate of commonwealth affairs,\n +You would say it hath been all in all his study:\n +List his discourse of war, and you shall hear\n +A fearful battle render\'d you in music:\n +Turn him to any cause of policy,\n +The Gordian knot of it he will unloose,\n +Familiar as his garter: that, when he speaks,\n +The air, a charter\'d libertine, is still,\n +And the mute wonder lurketh in men\'s ears,\n +To steal his sweet and honey\'d sentences;\n +So that the art and practic part of life\n +Must be the mistress to this theoric:\n +Which is a wonder how his grace should glean it,\n +Since his addiction was to courses vain,\n +His companies unletter\'d, rude and shallow,\n +His hours fill\'d up with riots, banquets, sports,\n +And never noted in him any study,\n +Any retirement, any sequestration\n +From open haunts and popularity.\n +\n +I come no more to make you laugh: things now,\n +That bear a weighty and a serious brow,\n +Sad, high, and working, full of state and woe,\n +Such noble scenes as draw the eye to flow,\n +e now present. Those that can pity, here\n +May, if they think it well, let fall a tear;\n +The subject will deserve it. Such as give\n +Their money out of hope they may believe,\n +May here find truth too. Those that come to see\n +Only a show or two, and so agree\n +The play may pass, if they be still and willing,\n +I\'ll undertake may see away their shilling\n +Richly in two short hours. Only they\n +That come to hear a merry bawdy play,\n +A noise of targets, or to see a fellow\n +In a long motley coat guarded with yellow,\n +Will be deceived; for, gentle hearers, know,\n +To rank our chosen truth with such a show\n +As fool and fight is, beside forfeiting\n +Our own brains, and the opinion that we bring,\n +To make that only true we now intend,\n +Will leave us never an understanding friend.\n +Therefore, for goodness\' sake, and as you are known\n +The first and happiest hearers of the town,\n +Be sad, as we would make ye: think ye see\n +The very persons of our noble story\n +As they were living; think you see them great,\n +And follow\'d with the general throng and sweat\n +Of thousand friends; then in a moment, see\n +How soon this mightiness meets misery:\n +And, if you can be merry then, I\'ll say\n +A man may weep upon his wedding-day.\n +\n +First, heaven be the record to my speech!\n +In the devotion of a subject\'s love,\n +Tendering the precious safety of my prince,\n +And free from other misbegotten hate,\n +Come I appellant to this princely presence.\n +Now, Thomas Mowbray, do I turn to thee,\n +And mark my greeting well; for what I speak\n +My body shall make good upon this earth,\n +Or my divine soul answer it in heaven.\n +Thou art a traitor and a miscreant,\n +Too good to be so and too bad to live,\n +Since the more fair and crystal is the sky,\n +The uglier seem the clouds that in it fly.\n +Once more, the more to aggravate the note,\n +With a foul traitor\'s name stuff I thy throat;\n +And wish, so please my sovereign, ere I move,\n +What my tongue speaks my right drawn sword may prove.\n +\n +Now is the winter of our discontent\n +Made glorious summer by this sun of York;\n +And all the clouds that lour\'d upon our house\n +In the deep bosom of the ocean buried.\n +Now are our brows bound with victorious wreaths;\n +Our bruised arms hung up for monuments;\n +Our stern alarums changed to merry meetings,\n +Our dreadful marches to delightful measures.\n +Grim-visaged war hath smooth\'d his wrinkled front;\n +And now, instead of mounting barded steeds\n +To fright the souls of fearful adversaries,\n +He capers nimbly in a lady\'s chamber\n +To the lascivious pleasing of a lute.\n +But I, that am not shaped for sportive tricks,\n +Nor made to court an amorous looking-glass;\n +I, that am rudely stamp\'d, and want love\'s majesty\n +To strut before a wanton ambling nymph;\n +I, that am curtail\'d of this fair proportion,\n +Cheated of feature by dissembling nature,\n +Deformed, unfinish\'d, sent before my time\n +Into this breathing world, scarce half made up,\n +And that so lamely and unfashionable\n +That dogs bark at me as I halt by them;\n +Why, I, in this weak piping time of peace,\n +Have no delight to pass away the time,\n +Unless to spy my shadow in the sun\n +And descant on mine own deformity:\n +And therefore, since I cannot prove a lover,\n +To entertain these fair well-spoken days,\n +I am determined to prove a villain\n +And hate the idle pleasures of these days.\n +Plots have I laid, inductions dangerous,\n +By drunken prophecies, libels and dreams,\n +To set my brother Clarence and the king\n +In deadly hate the one against the other:\n +And if King Edward be as true and just\n +As I am subtle, false and treacherous,\n +This day should Clarence closely be mew\'d up,\n +About a prophecy, which says that \'G\'\n +Of Edward\'s heirs the murderer shall be.\n +Dive, thoughts, down to my soul: here\n +Clarence comes.\n +\n +To bait fish withal: if it will feed nothing else,\n +it will feed my revenge. He hath disgraced me, and\n +hindered me half a million; laughed at my losses,\n +mocked at my gains, scorned my nation, thwarted my\n +bargains, cooled my friends, heated mine\n +enemies; and what\'s his reason? I am a Jew. Hath\n +not a Jew eyes? hath not a Jew hands, organs,\n +dimensions, senses, affections, passions? fed with\n +the same food, hurt with the same weapons, subject\n +to the same diseases, healed by the same means,\n +warmed and cooled by the same winter and summer, as\n +a Christian is? If you prick us, do we not bleed?\n +if you tickle us, do we not laugh? if you poison\n +us, do we not die? and if you wrong us, shall we not\n +revenge? If we are like you in the rest, we will\n +resemble you in that. If a Jew wrong a Christian,\n +what is his humility? Revenge. If a Christian\n +wrong a Jew, what should his sufferance be by\n +Christian example? Why, revenge. The villany you\n +teach me, I will execute, and it shall go hard but I\n +will better the instruction.\n +\n +Virtue! a fig! \'tis in ourselves that we are thus\n +or thus. Our bodies are our gardens, to the which\n +our wills are gardeners: so that if we will plant\n +nettles, or sow lettuce, set hyssop and weed up\n +thyme, supply it with one gender of herbs, or\n +distract it with many, either to have it sterile\n +with idleness, or manured with industry, why, the\n +power and corrigible authority of this lies in our\n +wills. If the balance of our lives had not one\n +scale of reason to poise another of sensuality, the\n +blood and baseness of our natures would conduct us\n +to most preposterous conclusions: but we have\n +reason to cool our raging motions, our carnal\n +stings, our unbitted lusts, whereof I take this that\n +you call love to be a sect or scion.\n +\n +Blow, winds, and crack your cheeks! rage! blow!\n +You cataracts and hurricanoes, spout\n +Till you have drench\'d our steeples, drown\'d the cocks!\n +You sulphurous and thought-executing fires,\n +Vaunt-couriers to oak-cleaving thunderbolts,\n +Singe my white head! And thou, all-shaking thunder,\n +Smite flat the thick rotundity o\' the world!\n +Crack nature\'s moulds, an germens spill at once,\n +That make ingrateful man! +5...\n +4...\n +3...\n +2...\n +1...\n +BEEEEEEEEEEEEEEEEEP! diff --git a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java index 339755f..005a483 100644 --- a/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java +++ b/tests/VoiceInteraction/src/com/android/test/voiceinteraction/AssistVisualizer.java @@ -38,6 +38,9 @@ public class AssistVisualizer extends View { final Matrix matrix; final String className; final CharSequence text; + final int scrollY; + final int[] lineCharOffsets; + final int[] lineBaselines; TextEntry(AssistStructure.ViewNode node, int parentLeft, int parentTop, Matrix matrix) { int left = parentLeft+node.getLeft(); @@ -48,16 +51,19 @@ public class AssistVisualizer extends View { this.matrix = new Matrix(matrix); this.className = node.getClassName(); this.text = node.getText() != null ? node.getText() : node.getContentDescription(); + this.scrollY = node.getScrollY(); + this.lineCharOffsets = node.getTextLineCharOffsets(); + this.lineBaselines = node.getTextLineBaselines(); } } AssistStructure mAssistStructure; final Paint mFramePaint = new Paint(); + final Paint mFrameBaselinePaint = new Paint(); final Paint mFrameNoTransformPaint = new Paint(); final ArrayList mMatrixStack = new ArrayList<>(); final ArrayList mTextRects = new ArrayList<>(); final int[] mTmpLocation = new int[2]; - final float[] mTmpMatrixPoint = new float[2]; public AssistVisualizer(Context context, @Nullable AttributeSet attrs) { super(context, attrs); @@ -65,6 +71,9 @@ public class AssistVisualizer extends View { mFramePaint.setColor(0xffff0000); mFramePaint.setStyle(Paint.Style.STROKE); mFramePaint.setStrokeWidth(0); + mFrameBaselinePaint.setColor(0xa0b0b000); + mFrameBaselinePaint.setStyle(Paint.Style.STROKE); + mFrameBaselinePaint.setStrokeWidth(0); float density = getResources().getDisplayMetrics().density; mFramePaint.setShadowLayer(density, density, density, 0xff000000); mFrameNoTransformPaint.setColor(0xff0000ff); @@ -106,6 +115,14 @@ public class AssistVisualizer extends View { + " in " + te.parentLeft + "," + te.parentTop + " matrix=" + te.matrix.toShortString() + ": " + te.text); + if (te.lineCharOffsets != null && te.lineBaselines != null) { + final int num = te.lineCharOffsets.length < te.lineBaselines.length + ? te.lineCharOffsets.length : te.lineBaselines.length; + for (int j=0; j