1 files changed, 56 insertions, 0 deletions
diff --git a/utility/Tokenizer.h b/utility/Tokenizer.h
new file mode 100644
index 0000000..de3f86c
--- /dev/null
+++ b/utility/Tokenizer.h
@@ -0,0 +1,56 @@
+///////////////////////////////////////////////////////////////////////////////
+// Tokenizer.h
+// ===========
+// General purpose string tokenizer (C++ string version)
+//
+// The default delimiters are space(" "), tab(\t, \v), newline(\n),
+// carriage return(\r), and form feed(\f).
+// If you want to use different delimiters, then use setDelimiter() to override
+// the delimiters. Note that the delimiter string can hold multiple characters.
+//
+//  AUTHOR: Song Ho Ahn (song.ahn@gmail.com)
+// CREATED: 2005-05-25
+// UPDATED: 2011-03-08
+///////////////////////////////////////////////////////////////////////////////
+
+#ifndef TOKENIZER_H
+#define TOKENIZER_H
+
+#include <string>
+#include <vector>
+
+// default delimiter string (space, tab, newline, carriage return, form feed)
+const std::string DEFAULT_DELIMITER = " \t\v\n\r\f";
+
+class Tokenizer
+{
+public:
+    // ctor/dtor
+    Tokenizer();
+    Tokenizer(const std::string& str, const std::string& delimiter=DEFAULT_DELIMITER);
+    ~Tokenizer();
+
+    // set string and delimiter
+    void set(const std::string& str, const std::string& delimiter=DEFAULT_DELIMITER);
+    void setString(const std::string& str);             // set source string only
+    void setDelimiter(const std::string& delimiter);    // set delimiter string only
+
+    std::string next();                                 // return the next token, return "" if it ends
+
+    std::vector<std::string> split();                   // return array of tokens from current cursor
+
+protected:
+
+
+private:
+    void skipDelimiter();                               // ignore leading delimiters
+    bool isDelimiter(char c);                           // check if the current char is delimiter
+
+    std::string buffer;                                 // input string
+    std::string token;                                  // output string
+    std::string delimiter;                              // delimiter string
+    std::string::const_iterator currPos;                // string iterator pointing the current position
+
+};
+
+#endif // TOKENIZER_H