Add useful functions to String8, which enables users to convert between UTF-8 and UTF-32

It will be used in SQL functions in external/sqlite/android.
See https://android-git.corp.google.com/g/Gerrit#change,5511 for example.

Related internal bug id: 1707173

1 files changed, 111 insertions, 5 deletions

diff --git a/include/utils/String8.h b/include/utils/String8.h
index c49faf6..ecc5774 100644
--- a/include/utils/String8.h
+++ b/include/utils/String8.h
@@ -29,11 +29,107 @@
 
 // ---------------------------------------------------------------------------
 
+extern "C" {
+
+typedef uint32_t char32_t;
+
+size_t strlen32(const char32_t *);
+size_t strnlen32(const char32_t *, size_t);
+
+/*
+ * Returns the length of "src" when "src" is valid UTF-8 string.
+ * Returns 0 if src is NULL, 0-length string or non UTF-8 string.
+ * This function should be used to determine whether "src" is valid UTF-8
+ * characters with valid unicode codepoints. "src" must be null-terminated.
+ *
+ * If you are going to use other GetUtf... functions defined in this header
+ * with string which may not be valid UTF-8 with valid codepoint (form 0 to
+ * 0x10FFFF), you should use this function before calling others, since the
+ * other functions do not check whether the string is valid UTF-8 or not.
+ *
+ * If you do not care whether "src" is valid UTF-8 or not, you should use
+ * strlen() as usual, which should be much faster.
+ */
+size_t utf8_length(const char *src);
+
+/*
+ * Returns the UTF-32 length of "src".
+ */
+size_t utf32_length(const char *src, size_t src_len);
+
+/*
+ * Returns the UTF-8 length of "src".
+ */
+size_t utf8_length_from_utf32(const char32_t *src, size_t src_len);
+
+/*
+ * Returns the unicode value at "index".
+ * Returns -1 when the index is invalid (equals to or more than "src_len").
+ * If returned value is positive, it is able to be converted to char32_t, which
+ * is unsigned. Then, if "next_index" is not NULL, the next index to be used is
+ * stored in "next_index". "next_index" can be NULL.
+ */
+int32_t utf32_at(const char *src, size_t src_len,
+                 size_t index, size_t *next_index);
+
+/*
+ * Stores a UTF-32 string converted from "src" in "dst", if "dst_length" is not
+ * large enough to store the string, the part of the "src" string is stored
+ * into "dst".
+ * Returns the size actually used for storing the string.
+ * "dst" is not null-terminated when dst_len is fully used (like strncpy).
+ */
+size_t utf8_to_utf32(const char* src, size_t src_len,
+                     char32_t* dst, size_t dst_len);
+
+/*
+ * Stores a UTF-8 string converted from "src" in "dst", if "dst_length" is not
+ * large enough to store the string, the part of the "src" string is stored
+ * into "dst" as much as possible. See the examples for more detail.
+ * Returns the size actually used for storing the string.
+ * dst" is not null-terminated when dst_len is fully used (like strncpy).
+ *
+ * Example 1
+ * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
+ * "src_len" == 2
+ * "dst_len" >= 7
+ * ->
+ * Returned value == 6
+ * "dst" becomes \xE3\x81\x82\xE3\x81\x84\0
+ * (note that "dst" is null-terminated)
+ *
+ * Example 2
+ * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
+ * "src_len" == 2
+ * "dst_len" == 5
+ * ->
+ * Returned value == 3
+ * "dst" becomes \xE3\x81\x82\0
+ * (note that "dst" is null-terminated, but \u3044 is not stored in "dst"
+ * since "dst" does not have enough size to store the character)
+ *
+ * Example 3
+ * "src" == \u3042\u3044 (\xE3\x81\x82\xE3\x81\x84)
+ * "src_len" == 2
+ * "dst_len" == 6
+ * ->
+ * Returned value == 6
+ * "dst" becomes \xE3\x81\x82\xE3\x81\x84
+ * (note that "dst" is NOT null-terminated, like strncpy)
+ */
+size_t utf32_to_utf8(const char32_t* src, size_t src_len,
+                     char* dst, size_t dst_len);
+
+}
+
+// ---------------------------------------------------------------------------
+
 namespace android {
 
 class TextOutput;
 
-//! This is a string holding UTF-8 characters.
+//! This is a string holding UTF-8 characters. Does not allow the value more
+// than 0x10FFFF, which is not valid unicode codepoint.
 class String8
 {
 public:
@@ -45,7 +141,8 @@ public:
     explicit                    String8(const String16& o);
     explicit                    String8(const char16_t* o);
     explicit                    String8(const char16_t* o, size_t numChars);
-    
+    explicit                    String8(const char32_t* o);
+    explicit                    String8(const char32_t* o, size_t numChars);
                                 ~String8();
     
     inline  const char*         string() const;
@@ -59,11 +156,20 @@ public:
             status_t            setTo(const char* other);
             status_t            setTo(const char* other, size_t numChars);
             status_t            setTo(const char16_t* other, size_t numChars);
-    
+            status_t            setTo(const char32_t* other,
+                                      size_t length);
+
             status_t            append(const String8& other);
             status_t            append(const char* other);
             status_t            append(const char* other, size_t numChars);
 
+            // Note that this function takes O(N) time to calculate the value.
+            // No cache value is stored.
+            size_t              getUtf32Length() const;
+            int32_t             getUtf32At(size_t index,
+                                           size_t *next_index) const;
+            size_t              getUtf32(char32_t* dst, size_t dst_len) const;
+
     inline  String8&            operator=(const String8& other);
     inline  String8&            operator=(const char* other);
     
@@ -103,7 +209,7 @@ public:
             void                toLower(size_t start, size_t numChars);
             void                toUpper();
             void                toUpper(size_t start, size_t numChars);
-            
+
     /*
      * These methods operate on the string as if it were a path name.
      */
@@ -346,7 +452,7 @@ inline String8::operator const char*() const
     return mString;
 }
 
-}; // namespace android
+}  // namespace android
 
 // ---------------------------------------------------------------------------