diff options
Diffstat (limited to 'utility')
| -rw-r--r-- | utility/Android.mk | 29 | ||||
| -rw-r--r-- | utility/Tokenizer.cpp | 125 | ||||
| -rw-r--r-- | utility/Tokenizer.h | 56 |
3 files changed, 210 insertions, 0 deletions
diff --git a/utility/Android.mk b/utility/Android.mk new file mode 100644 index 0000000..f6af488 --- /dev/null +++ b/utility/Android.mk @@ -0,0 +1,29 @@ +LOCAL_PATH:= $(call my-dir) + +include $(CLEAR_VARS) + + +LOCAL_SRC_FILES:= Tokenizer.cpp + +LOCAL_MODULE:= libutility + +LOCAL_MODULE_TAGS := + + +LOCAL_C_INCLUDES += + + +LOCAL_C_INCLUDES += \ + external/stlport/stlport/ \ + bionic/libstdc++ \ + bionic/ + +LOCAL_C_INCLUDES += + +LOCAL_SHARED_LIBRARIES := + +LOCAL_LDLIBS += + + +include $(BUILD_STATIC_LIBRARY) + diff --git a/utility/Tokenizer.cpp b/utility/Tokenizer.cpp new file mode 100644 index 0000000..9ea4ea4 --- /dev/null +++ b/utility/Tokenizer.cpp @@ -0,0 +1,125 @@ +///////////////////////////////////////////////////////////////////////////////
+// Tokenizer.cpp
+// =============
+// General purpose string tokenizer (C++ string version)
+//
+// The default delimiters are space(" "), tab(\t, \v), newline(\n),
+// carriage return(\r), and form feed(\f).
+// If you want to use different delimiters, then use setDelimiter() to override
+// the delimiters. Note that the delimiter string can hold multiple characters.
+//
+// AUTHOR: Song Ho Ahn (song.ahn@gmail.com)
+// CREATED: 2005-05-25
+// UPDATED: 2011-03-08
+///////////////////////////////////////////////////////////////////////////////
+
+#include "Tokenizer.h"
+
+
+///////////////////////////////////////////////////////////////////////////////
+// constructor
+///////////////////////////////////////////////////////////////////////////////
+Tokenizer::Tokenizer() : buffer(""), token(""), delimiter(DEFAULT_DELIMITER)
+{
+ currPos = buffer.begin();
+}
+
+Tokenizer::Tokenizer(const std::string& str, const std::string& delimiter) : buffer(str), token(""), delimiter(delimiter)
+{
+ currPos = buffer.begin();
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+// destructor
+///////////////////////////////////////////////////////////////////////////////
+Tokenizer::~Tokenizer()
+{
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+// reset string buffer, delimiter and the currsor position
+///////////////////////////////////////////////////////////////////////////////
+void Tokenizer::set(const std::string& str, const std::string& delimiter)
+{
+ this->buffer = str;
+ this->delimiter = delimiter;
+ this->currPos = buffer.begin();
+}
+
+void Tokenizer::setString(const std::string& str)
+{
+ this->buffer = str;
+ this->currPos = buffer.begin();
+}
+
+void Tokenizer::setDelimiter(const std::string& delimiter)
+{
+ this->delimiter = delimiter;
+ this->currPos = buffer.begin();
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+// return the next token
+// If cannot find a token anymore, return "".
+///////////////////////////////////////////////////////////////////////////////
+std::string Tokenizer::next()
+{
+ if(buffer.size() <= 0) return ""; // skip if buffer is empty
+
+ token.clear(); // reset token string
+
+ this->skipDelimiter(); // skip leading delimiters
+
+ // append each char to token string until it meets delimiter
+ while(currPos != buffer.end() && !isDelimiter(*currPos))
+ {
+ token += *currPos;
+ ++currPos;
+ }
+ return token;
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+// skip ang leading delimiters
+///////////////////////////////////////////////////////////////////////////////
+void Tokenizer::skipDelimiter()
+{
+ while(currPos != buffer.end() && isDelimiter(*currPos))
+ ++currPos;
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+// return true if the current character is delimiter
+///////////////////////////////////////////////////////////////////////////////
+bool Tokenizer::isDelimiter(char c)
+{
+ return (delimiter.find(c) != std::string::npos);
+}
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+// split the input string into multiple tokens
+// This function scans tokens from the current cursor position.
+///////////////////////////////////////////////////////////////////////////////
+std::vector<std::string> Tokenizer::split()
+{
+ std::vector<std::string> tokens;
+ std::string token;
+ while((token = this->next()) != "")
+ {
+ tokens.push_back(token);
+ }
+
+ return tokens;
+}
diff --git a/utility/Tokenizer.h b/utility/Tokenizer.h new file mode 100644 index 0000000..de3f86c --- /dev/null +++ b/utility/Tokenizer.h @@ -0,0 +1,56 @@ +///////////////////////////////////////////////////////////////////////////////
+// Tokenizer.h
+// ===========
+// General purpose string tokenizer (C++ string version)
+//
+// The default delimiters are space(" "), tab(\t, \v), newline(\n),
+// carriage return(\r), and form feed(\f).
+// If you want to use different delimiters, then use setDelimiter() to override
+// the delimiters. Note that the delimiter string can hold multiple characters.
+//
+// AUTHOR: Song Ho Ahn (song.ahn@gmail.com)
+// CREATED: 2005-05-25
+// UPDATED: 2011-03-08
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef TOKENIZER_H
+#define TOKENIZER_H
+
+#include <string>
+#include <vector> +
+// default delimiter string (space, tab, newline, carriage return, form feed)
+const std::string DEFAULT_DELIMITER = " \t\v\n\r\f";
+
+class Tokenizer
+{
+public:
+ // ctor/dtor
+ Tokenizer();
+ Tokenizer(const std::string& str, const std::string& delimiter=DEFAULT_DELIMITER);
+ ~Tokenizer();
+
+ // set string and delimiter
+ void set(const std::string& str, const std::string& delimiter=DEFAULT_DELIMITER);
+ void setString(const std::string& str); // set source string only
+ void setDelimiter(const std::string& delimiter); // set delimiter string only
+
+ std::string next(); // return the next token, return "" if it ends
+
+ std::vector<std::string> split(); // return array of tokens from current cursor
+
+protected:
+
+
+private:
+ void skipDelimiter(); // ignore leading delimiters
+ bool isDelimiter(char c); // check if the current char is delimiter
+
+ std::string buffer; // input string
+ std::string token; // output string
+ std::string delimiter; // delimiter string
+ std::string::const_iterator currPos; // string iterator pointing the current position
+
+};
+
+#endif // TOKENIZER_H
|