Full text search: Asian language support

Bug: 2078420
Change-Id: I1a99ad05bb6e30b539134d60f669134b87dbf180

8 files changed, 162 insertions, 56 deletions

diff --git a/src/com/android/providers/contacts/ContactsDatabaseHelper.java b/src/com/android/providers/contacts/ContactsDatabaseHelper.java
index 51829ec..a772201 100644
--- a/src/com/android/providers/contacts/ContactsDatabaseHelper.java
+++ b/src/com/android/providers/contacts/ContactsDatabaseHelper.java
@@ -92,7 +92,7 @@ import java.util.Locale;
      *   500-599 Honeycomb-MR1
      * </pre>
      */
-    static final int DATABASE_VERSION = 501;
+    static final int DATABASE_VERSION = 502;
 
     private static final String DATABASE_NAME = "contacts2.db";
     private static final String DATABASE_PRESENCE = "presence_db";
@@ -392,11 +392,9 @@ import java.util.Locale;
         public static final int NAME_COLLATION_KEY = 2;
         public static final int NICKNAME = 3;
         public static final int EMAIL_BASED_NICKNAME = 4;
-        public static final int NAME_SHORTHAND = 6;
-        public static final int NAME_CONSONANTS = 7;
 
         // This is the highest name lookup type code plus one
-        public static final int TYPE_COUNT = 8;
+        public static final int TYPE_COUNT = 5;
 
         public static boolean isBasedOnStructuredName(int nameLookupType) {
             return nameLookupType == NameLookupType.NAME_EXACT
@@ -1779,6 +1777,11 @@ import java.util.Locale;
             oldVersion = 501;
         }
 
+        if (oldVersion < 502) {
+            upgradeToVersion502(db);
+            oldVersion = 502;
+        }
+
         if (upgradeViewsAndTriggers) {
             createContactsViews(db);
             createGroupsView(db);
@@ -2805,6 +2808,12 @@ import java.util.Locale;
         setProperty(db, SearchIndexManager.PROPERTY_SEARCH_INDEX_VERSION, "0");
     }
 
+    private void upgradeToVersion502(SQLiteDatabase db) {
+        // Remove Chinese and Korean name lookup - this data is now in the search index
+        db.execSQL("DELETE FROM name_lookup WHERE name_type IN (6, 7)");
+        setProperty(db, SearchIndexManager.PROPERTY_SEARCH_INDEX_VERSION, "0");
+    }
+
     public String extractHandleFromEmailAddress(String email) {
         Rfc822Token[] tokens = Rfc822Tokenizer.tokenize(email);
         if (tokens.length == 0) {
@@ -3919,13 +3928,6 @@ import java.util.Locale;
             insertNameLookup(rawContactId, dataId, NameLookupType.NAME_COLLATION_KEY,
                     NameNormalizer.normalize(mSb.toString()));
         }
-
-        if (givenName != null) {
-            // We want the phonetic given name to be used for search, but not for aggregation,
-            // which is why we are using NAME_SHORTHAND rather than NAME_COLLATION_KEY
-            insertNameLookup(rawContactId, dataId, NameLookupType.NAME_SHORTHAND,
-                    NameNormalizer.normalize(givenName.trim()));
-        }
     }
 
     /**
diff --git a/src/com/android/providers/contacts/ContactsProvider2.java b/src/com/android/providers/contacts/ContactsProvider2.java
index 1cf76b0..1a8178d 100644
--- a/src/com/android/providers/contacts/ContactsProvider2.java
+++ b/src/com/android/providers/contacts/ContactsProvider2.java
@@ -417,9 +417,7 @@ public class ContactsProvider2 extends SQLiteContentProvider implements OnAccoun
     private static final String CONTACT_LOOKUP_NAME_TYPES =
             NameLookupType.NAME_COLLATION_KEY + "," +
             NameLookupType.EMAIL_BASED_NICKNAME + "," +
-            NameLookupType.NICKNAME + "," +
-            NameLookupType.NAME_SHORTHAND + "," +
-            NameLookupType.NAME_CONSONANTS;
+            NameLookupType.NICKNAME;
 
     /**
      * If any of these columns are used in a Data projection, there is no point in
@@ -5052,9 +5050,7 @@ public class ContactsProvider2 extends SQLiteContentProvider implements OnAccoun
             sb.append(normalizedName);
             sb.append("*' AND " + NameLookupColumns.NAME_TYPE + " IN ("
                     + NameLookupType.NAME_COLLATION_KEY + ","
-                    + NameLookupType.NICKNAME + ","
-                    + NameLookupType.NAME_SHORTHAND + ","
-                    + NameLookupType.NAME_CONSONANTS);
+                    + NameLookupType.NICKNAME);
             if (allowEmailMatch) {
                 sb.append("," + NameLookupType.EMAIL_BASED_NICKNAME);
             }
diff --git a/src/com/android/providers/contacts/DataRowHandlerForStructuredName.java b/src/com/android/providers/contacts/DataRowHandlerForStructuredName.java
index 3411d0b..55aeec8 100644
--- a/src/com/android/providers/contacts/DataRowHandlerForStructuredName.java
+++ b/src/com/android/providers/contacts/DataRowHandlerForStructuredName.java
@@ -191,11 +191,12 @@ public class DataRowHandlerForStructuredName extends DataRowHandler {
 
     @Override
     public void appendSearchableData(IndexBuilder builder) {
-        builder.appendTokenFromColumn(StructuredName.PREFIX);
-        builder.appendTokenFromColumn(StructuredName.FAMILY_NAME);
-        builder.appendTokenFromColumn(StructuredName.GIVEN_NAME);
-        builder.appendTokenFromColumn(StructuredName.MIDDLE_NAME);
-        builder.appendTokenFromColumn(StructuredName.SUFFIX);
+        String name = builder.getString(StructuredName.DISPLAY_NAME);
+        Integer fullNameStyle = builder.getInt(StructuredName.FULL_NAME_STYLE);
+
+        mNameLookupBuilder.appendToSearchIndex(builder, name, fullNameStyle != null
+                        ? mSplitter.getAdjustedFullNameStyle(fullNameStyle)
+                        : FullNameStyle.UNDEFINED);
 
         String phoneticFamily = builder.getString(StructuredName.PHONETIC_FAMILY_NAME);
         String phoneticMiddle = builder.getString(StructuredName.PHONETIC_MIDDLE_NAME);
@@ -219,6 +220,5 @@ public class DataRowHandlerForStructuredName extends DataRowHandler {
             }
             builder.appendToken(mSb.toString().trim());
         }
-
     }
 }
diff --git a/src/com/android/providers/contacts/GlobalSearchSupport.java b/src/com/android/providers/contacts/GlobalSearchSupport.java
index 6ace38d..7b9d4a6 100644
--- a/src/com/android/providers/contacts/GlobalSearchSupport.java
+++ b/src/com/android/providers/contacts/GlobalSearchSupport.java
@@ -319,7 +319,7 @@ public class GlobalSearchSupport {
         int to = snippet.length();
         int start = snippet.indexOf(SNIPPET_START_MATCH);
         if (start == -1) {
-            return snippet;
+            return null;
         }
 
         int firstNl = snippet.lastIndexOf('\n', start);
diff --git a/src/com/android/providers/contacts/NameLookupBuilder.java b/src/com/android/providers/contacts/NameLookupBuilder.java
index 46ce481..68c84e8 100644
--- a/src/com/android/providers/contacts/NameLookupBuilder.java
+++ b/src/com/android/providers/contacts/NameLookupBuilder.java
@@ -17,6 +17,7 @@
 package com.android.providers.contacts;
 
 import com.android.providers.contacts.ContactsDatabaseHelper.NameLookupType;
+import com.android.providers.contacts.SearchIndexManager.IndexBuilder;
 
 import android.provider.ContactsContract.FullNameStyle;
 
@@ -136,31 +137,42 @@ public abstract class NameLookupBuilder {
 
         insertNameVariants(rawContactId, dataId, 0, tokenCount, !tooManyTokens, true);
         insertNicknamePermutations(rawContactId, dataId, 0, tokenCount);
-        insertNameShorthandLookup(rawContactId, dataId, name, fullNameStyle);
-        insertNameLookupForLocaleBasedName(rawContactId, dataId, name, fullNameStyle);
+    }
+
+    public void appendToSearchIndex(IndexBuilder builder, String name, int fullNameStyle) {
+        int tokenCount = mSplitter.tokenize(mNames, name);
+        if (tokenCount == 0) {
+            return;
+        }
+
+        for (int i = 0; i < tokenCount; i++) {
+            builder.appendToken(mNames[i]);
+        }
+
+        appendNameShorthandLookup(builder, name, fullNameStyle);
+        appendNameLookupForLocaleBasedName(builder, name, fullNameStyle);
     }
 
     /**
      * Insert more name indexes according to locale specifies.
      */
-    private void insertNameLookupForLocaleBasedName(long rawContactId, long dataId,
+    private void appendNameLookupForLocaleBasedName(IndexBuilder builder,
             String fullName, int fullNameStyle) {
         if (fullNameStyle == FullNameStyle.KOREAN) {
             NameSplitter.Name name = new NameSplitter.Name();
             mSplitter.split(name, fullName, fullNameStyle);
             if (name.givenNames != null) {
-                insertNameLookup(rawContactId, dataId, NameLookupType.NAME_SHORTHAND,
-                        normalizeName(name.givenNames));
-                insertKoreanNameConsonantsLookup(rawContactId, dataId, name.givenNames);
+                builder.appendToken(name.givenNames);
+                appendKoreanNameConsonantsLookup(builder, name.givenNames);
             }
-            insertKoreanNameConsonantsLookup(rawContactId, dataId, fullName);
+            appendKoreanNameConsonantsLookup(builder, fullName);
         }
     }
 
     /**
      * Inserts Korean lead consonants records of name for the given structured name.
      */
-    private void insertKoreanNameConsonantsLookup(long rawContactId, long dataId, String name) {
+    private void appendKoreanNameConsonantsLookup(IndexBuilder builder, String name) {
         int position = 0;
         int consonantLength = 0;
         int character;
@@ -206,8 +218,7 @@ public abstract class NameLookupBuilder {
         // At least, insert consonants when Korean characters are two or more.
         // Only one character cases are covered by NAME_COLLATION_KEY
         if (consonantLength > 1) {
-            insertNameLookup(rawContactId, dataId, NameLookupType.NAME_CONSONANTS,
-                    normalizeName(mStringBuilder.toString()));
+            builder.appendToken(mStringBuilder.toString());
         }
     }
 
@@ -308,15 +319,12 @@ public abstract class NameLookupBuilder {
         }
     }
 
-    private void insertNameShorthandLookup(long rawContactId, long dataId, String name,
-            int fullNameStyle) {
+    private void appendNameShorthandLookup(IndexBuilder builder, String name, int fullNameStyle) {
         Iterator<String> it =
                 ContactLocaleUtils.getIntance().getNameLookupKeys(name, fullNameStyle);
         if (it != null) {
             while (it.hasNext()) {
-                String key = it.next();
-                insertNameLookup(rawContactId, dataId, NameLookupType.NAME_SHORTHAND,
-                        normalizeName(key));
+                builder.appendToken(it.next());
             }
         }
     }
diff --git a/tests/src/com/android/providers/contacts/BaseContactsProvider2Test.java b/tests/src/com/android/providers/contacts/BaseContactsProvider2Test.java
index 55f38c9..a62f9c5 100644
--- a/tests/src/com/android/providers/contacts/BaseContactsProvider2Test.java
+++ b/tests/src/com/android/providers/contacts/BaseContactsProvider2Test.java
@@ -776,6 +776,8 @@ public abstract class BaseContactsProvider2Test extends AndroidTestCase {
         String value = null;
         Cursor c = mResolver.query(uri, new String[] { column }, selection, selectionArgs, null);
         try {
+            assertEquals("Record count", 1, c.getCount());
+
             if (c.moveToFirst()) {
                 value = c.getString(c.getColumnIndex(column));
             }
diff --git a/tests/src/com/android/providers/contacts/NameLookupBuilderTest.java b/tests/src/com/android/providers/contacts/NameLookupBuilderTest.java
index 91ab761..a5cff7a 100644
--- a/tests/src/com/android/providers/contacts/NameLookupBuilderTest.java
+++ b/tests/src/com/android/providers/contacts/NameLookupBuilderTest.java
@@ -191,17 +191,12 @@ public class NameLookupBuilderTest extends TestCase {
         mBuilder.insertNameLookup(0, 0, "\u695A\u8FAD", FullNameStyle.CHINESE);
         assertEquals(
                 "(0:\u695A\u8FAD)" +
-                "(2:\u695A\u8FAD)" +
-                "(6:\u695A\u8FAD)" +
-                "(6:CI)" +
-                "(6:\u8FAD)" +
-                "(6:CHUCI)" +
-                "(6:CC)" +
-                "(6:C)", mBuilder.inserted());
+                "(2:\u695A\u8FAD)",
+                mBuilder.inserted());
     }
 
     public void testKoreanName() {
-        // Only run this test when Chinese collation is supported.
+        // Only run this test when Korean collation is supported.
         if (!Arrays.asList(Collator.getAvailableLocales()).contains(Locale.KOREA)) {
             return;
         }
@@ -210,10 +205,7 @@ public class NameLookupBuilderTest extends TestCase {
         mBuilder.insertNameLookup(0, 0, "\uC774\uC0C1\uC77C", FullNameStyle.KOREAN);
         assertEquals(
                 "(0:\uC774\uC0C1\uC77C)" + // Lee Sang Il
-                "(2:\uC774\uC0C1\uC77C)" + // Lee Sang Il
-                "(6:\uC0C1\uC77C)" + // Sang Il : given name
-                "(7:\u1109\u110B)" + // SIOS IEUNG : consonants of given name
-                "(7:\u110B\u1109\u110B)", // RIEUL SIOS IEUNG : consonants of fullname
+                "(2:\uC774\uC0C1\uC77C)",
                 mBuilder.inserted());
     }
 
@@ -227,10 +219,7 @@ public class NameLookupBuilderTest extends TestCase {
         mBuilder.insertNameLookup(0, 0, "\uC120\uC6B0\uC6A9\uB140", FullNameStyle.KOREAN);
         assertEquals(
                 "(0:\uC120\uC6B0\uC6A9\uB140)" + // Sun Woo Young Nyeu
-                "(2:\uC120\uC6B0\uC6A9\uB140)" + // Sun Woo Young Nyeu
-                "(6:\uC6A9\uB140)" + // Young Nyeu : given name
-                "(7:\u110B\u1102)" + // IEUNG NIEUN : consonants of given name
-                "(7:\u1109\u110B\u110B\u1102)", // SIOS IEUNG IEUNG NIEUN : consonants of fullname
+                "(2:\uC120\uC6B0\uC6A9\uB140)",  // Sun Woo Young Nyeu
                 mBuilder.inserted());
     }
 
diff --git a/tests/src/com/android/providers/contacts/SearchIndexManagerTest.java b/tests/src/com/android/providers/contacts/SearchIndexManagerTest.java
index 09e7924..52ed18f 100644
--- a/tests/src/com/android/providers/contacts/SearchIndexManagerTest.java
+++ b/tests/src/com/android/providers/contacts/SearchIndexManagerTest.java
@@ -27,6 +27,10 @@ import android.provider.ContactsContract.Contacts;
 import android.provider.ContactsContract.SearchSnippetColumns;
 import android.test.suitebuilder.annotation.MediumTest;
 
+import java.text.Collator;
+import java.util.Arrays;
+import java.util.Locale;
+
 /**
  * Unit tests for {@link SearchIndexManager}.
  *
@@ -57,7 +61,112 @@ public class SearchIndexManagerTest extends BaseContactsProvider2Test {
         insertStructuredName(rawContactId, values);
 
         assertSearchIndex(
-                contactId, null, "Doe John Parr Bob I. Mrs. Parr Helen I. PhD par helen parhelen");
+                contactId, null, "John Doe Bob I Parr Helen I Parr PhD par helen parhelen");
+    }
+
+    public void testSearchIndexForChineseName() {
+        // Only run this test when Chinese collation is supported
+        if (!Arrays.asList(Collator.getAvailableLocales()).contains(Locale.CHINA)) {
+            return;
+        }
+
+        long rawContactId = createRawContact();
+        long contactId = queryContactId(rawContactId);
+        ContentValues values = new ContentValues();
+        values.put(StructuredName.DISPLAY_NAME, "\u695A\u8FAD");    // CHUCI
+        insertStructuredName(rawContactId, values);
+
+        assertSearchIndex(
+                contactId, null, "\u695A\u8FAD \u695A\u8FAD CI \u8FAD CHUCI CC C");
+    }
+
+    public void testSearchByChineseName() {
+        // Only run this test when Chinese collation is supported
+        if (!Arrays.asList(Collator.getAvailableLocales()).contains(Locale.CHINA)) {
+            return;
+        }
+
+        long rawContactId = createRawContact();
+        long contactId = queryContactId(rawContactId);
+        ContentValues values = new ContentValues();
+        values.put(StructuredName.DISPLAY_NAME, "\u695A\u8FAD");    // CHUCI
+        insertStructuredName(rawContactId, values);
+
+        assertStoredValue(buildSearchUri("\u695A\u8FAD"), SearchSnippetColumns.SNIPPET, null);
+        assertStoredValue(buildSearchUri("\u8FAD"), SearchSnippetColumns.SNIPPET, null);
+        assertStoredValue(buildSearchUri("CI"), SearchSnippetColumns.SNIPPET, null);
+        assertStoredValue(buildSearchUri("CHUCI"), SearchSnippetColumns.SNIPPET, null);
+        assertStoredValue(buildSearchUri("CC"), SearchSnippetColumns.SNIPPET, null);
+        assertStoredValue(buildSearchUri("C"), SearchSnippetColumns.SNIPPET, null);
+    }
+
+    public void testSearchIndexForKoreanName() {
+        // Only run this test when Korean collation is supported
+        if (!Arrays.asList(Collator.getAvailableLocales()).contains(Locale.KOREA)) {
+            return;
+        }
+
+        long rawContactId = createRawContact();
+        long contactId = queryContactId(rawContactId);
+        ContentValues values = new ContentValues();
+        values.put(StructuredName.DISPLAY_NAME, "\uC774\uC0C1\uC77C");    // Lee Sang Il
+        insertStructuredName(rawContactId, values);
+
+        assertSearchIndex(
+                contactId, null, "\uC774\uC0C1\uC77C \uC0C1\uC77C \u1109\u110B \u110B\u1109\u110B");
+    }
+
+    public void testSearchByKoreanName() {
+        // Only run this test when Korean collation is supported
+        if (!Arrays.asList(Collator.getAvailableLocales()).contains(Locale.KOREA)) {
+            return;
+        }
+
+        long rawContactId = createRawContact();
+        ContentValues values = new ContentValues();
+        values.put(StructuredName.DISPLAY_NAME, "\uC774\uC0C1\uC77C");   // Lee Sang Il
+        insertStructuredName(rawContactId, values);
+
+        // Full name: Lee Sang Il
+        assertStoredValue(buildSearchUri("\uC774\uC0C1\uC77C"), SearchSnippetColumns.SNIPPET, null);
+
+        // Given name: Sang Il
+        assertStoredValue(buildSearchUri("\uC0C1\uC77C"), SearchSnippetColumns.SNIPPET, null);
+
+        // Consonants of given name: SIOS IEUNG
+        assertStoredValue(buildSearchUri("\u1109\u110B"), SearchSnippetColumns.SNIPPET, null);
+
+        // Consonants of full name: RIEUL SIOS IEUNG
+        assertStoredValue(buildSearchUri("\u110B\u1109\u110B"), SearchSnippetColumns.SNIPPET, null);
+    }
+
+    public void testSearchByKoreanNameWithTwoCharactersFamilyName() {
+        // Only run this test when Korean collation is supported.
+        if (!Arrays.asList(Collator.getAvailableLocales()).contains(Locale.KOREA)) {
+            return;
+        }
+
+        long rawContactId = createRawContact();
+
+        // Sun Woo Young Nyeu
+        ContentValues values = new ContentValues();
+        values.put(StructuredName.DISPLAY_NAME, "\uC120\uC6B0\uC6A9\uB140");
+
+        insertStructuredName(rawContactId, values);
+
+        // Full name: Sun Woo Young Nyeu
+        assertStoredValue(
+                buildSearchUri("\uC120\uC6B0\uC6A9\uB140"), SearchSnippetColumns.SNIPPET, null);
+
+        // Given name: Young Nyeu
+        assertStoredValue(buildSearchUri("\uC6A9\uB140"), SearchSnippetColumns.SNIPPET, null);
+
+        // Consonants of given name: IEUNG NIEUN
+        assertStoredValue(buildSearchUri("\u110B\u1102"), SearchSnippetColumns.SNIPPET, null);
+
+        // Consonants of full name: SIOS IEUNG IEUNG NIEUN
+        assertStoredValue(
+                buildSearchUri("\u1109\u110B\u110B\u1102"), SearchSnippetColumns.SNIPPET, null);
     }
 
     public void testSearchIndexForOrganization() {