summaryrefslogtreecommitdiffstats
path: root/tools/localize
diff options
context:
space:
mode:
authorThe Android Open Source Project <initial-contribution@android.com>2009-03-03 19:31:44 -0800
committerThe Android Open Source Project <initial-contribution@android.com>2009-03-03 19:31:44 -0800
commit9066cfe9886ac131c34d59ed0e2d287b0e3c0087 (patch)
treed88beb88001f2482911e3d28e43833b50e4b4e97 /tools/localize
parentd83a98f4ce9cfa908f5c54bbd70f03eec07e7553 (diff)
downloadframeworks_base-9066cfe9886ac131c34d59ed0e2d287b0e3c0087.zip
frameworks_base-9066cfe9886ac131c34d59ed0e2d287b0e3c0087.tar.gz
frameworks_base-9066cfe9886ac131c34d59ed0e2d287b0e3c0087.tar.bz2
auto import from //depot/cupcake/@135843
Diffstat (limited to 'tools/localize')
-rw-r--r--tools/localize/Android.mk56
-rw-r--r--tools/localize/Configuration.cpp76
-rw-r--r--tools/localize/Configuration.h38
-rw-r--r--tools/localize/Perforce.cpp230
-rw-r--r--tools/localize/Perforce.h25
-rw-r--r--tools/localize/Perforce_test.cpp62
-rw-r--r--tools/localize/SourcePos.cpp166
-rw-r--r--tools/localize/SourcePos.h28
-rw-r--r--tools/localize/Values.cpp134
-rw-r--r--tools/localize/Values.h48
-rw-r--r--tools/localize/ValuesFile.cpp266
-rw-r--r--tools/localize/ValuesFile.h52
-rw-r--r--tools/localize/ValuesFile_test.cpp54
-rw-r--r--tools/localize/XLIFFFile.cpp609
-rw-r--r--tools/localize/XLIFFFile.h98
-rw-r--r--tools/localize/XLIFFFile_test.cpp115
-rw-r--r--tools/localize/XMLHandler.cpp793
-rw-r--r--tools/localize/XMLHandler.h197
-rw-r--r--tools/localize/XMLHandler_test.cpp133
-rw-r--r--tools/localize/XMLNode.h19
-rw-r--r--tools/localize/file_utils.cpp143
-rw-r--r--tools/localize/file_utils.h21
-rw-r--r--tools/localize/localize.cpp767
-rw-r--r--tools/localize/localize.h40
-rw-r--r--tools/localize/localize_test.cpp219
-rw-r--r--tools/localize/log.h7
-rw-r--r--tools/localize/merge_res_and_xliff.cpp391
-rw-r--r--tools/localize/merge_res_and_xliff.h13
-rw-r--r--tools/localize/merge_res_and_xliff_test.cpp47
-rw-r--r--tools/localize/res_check.cpp106
-rw-r--r--tools/localize/res_check.h12
-rw-r--r--tools/localize/test.cpp31
-rw-r--r--tools/localize/testdata/config.xml15
-rw-r--r--tools/localize/testdata/import.xliff72
-rw-r--r--tools/localize/testdata/merge.xliff72
-rw-r--r--tools/localize/testdata/merge_en_current.xml44
-rw-r--r--tools/localize/testdata/merge_en_old.xml45
-rw-r--r--tools/localize/testdata/merge_xx_current.xml22
-rw-r--r--tools/localize/testdata/merge_xx_old.xml21
-rw-r--r--tools/localize/testdata/pseudo.xliff40
-rw-r--r--tools/localize/testdata/res/values-zz-rZZ/strings.xml22
-rw-r--r--tools/localize/testdata/res/values/strings.xml44
-rw-r--r--tools/localize/testdata/strip_xliff.xliff70
-rw-r--r--tools/localize/testdata/values/strings.xml32
-rw-r--r--tools/localize/testdata/xliff1.xliff46
-rw-r--r--tools/localize/testdata/xml.xml16
-rw-r--r--tools/localize/xmb.cpp181
-rw-r--r--tools/localize/xmb.h11
48 files changed, 5749 insertions, 0 deletions
diff --git a/tools/localize/Android.mk b/tools/localize/Android.mk
new file mode 100644
index 0000000..186177f
--- /dev/null
+++ b/tools/localize/Android.mk
@@ -0,0 +1,56 @@
+#
+# Copyright 2006 The Android Open Source Project
+#
+# Android Asset Packaging Tool
+#
+
+LOCAL_PATH:= $(call my-dir)
+include $(CLEAR_VARS)
+
+LOCAL_SRC_FILES := \
+ file_utils.cpp \
+ localize.cpp \
+ merge_res_and_xliff.cpp \
+ res_check.cpp \
+ xmb.cpp \
+ Configuration.cpp \
+ Perforce.cpp \
+ SourcePos.cpp \
+ Values.cpp \
+ ValuesFile.cpp \
+ XLIFFFile.cpp \
+ XMLHandler.cpp
+
+LOCAL_C_INCLUDES := \
+ external/expat/lib \
+ build/libs/host/include
+
+LOCAL_CFLAGS += -g -O0
+
+LOCAL_STATIC_LIBRARIES := \
+ libexpat \
+ libhost \
+ libutils \
+ libcutils
+
+ifeq ($(HOST_OS),linux)
+LOCAL_LDLIBS += -lrt
+endif
+
+
+LOCAL_MODULE := localize
+
+ifeq (a,a)
+ LOCAL_CFLAGS += -DLOCALIZE_WITH_TESTS
+ LOCAL_SRC_FILES += \
+ test.cpp \
+ localize_test.cpp \
+ merge_res_and_xliff_test.cpp \
+ Perforce_test.cpp \
+ ValuesFile_test.cpp \
+ XLIFFFile_test.cpp \
+ XMLHandler_test.cpp
+endif
+
+include $(BUILD_HOST_EXECUTABLE)
+
diff --git a/tools/localize/Configuration.cpp b/tools/localize/Configuration.cpp
new file mode 100644
index 0000000..56addbd
--- /dev/null
+++ b/tools/localize/Configuration.cpp
@@ -0,0 +1,76 @@
+#include "Configuration.h"
+#include <string.h>
+
+int
+Configuration::Compare(const Configuration& that) const
+{
+ int n;
+
+ n = locale.compare(that.locale);
+ if (n != 0) return n;
+
+ n = vendor.compare(that.vendor);
+ if (n != 0) return n;
+
+ n = orientation.compare(that.orientation);
+ if (n != 0) return n;
+
+ n = density.compare(that.density);
+ if (n != 0) return n;
+
+ n = touchscreen.compare(that.touchscreen);
+ if (n != 0) return n;
+
+ n = keyboard.compare(that.keyboard);
+ if (n != 0) return n;
+
+ n = navigation.compare(that.navigation);
+ if (n != 0) return n;
+
+ n = screenSize.compare(that.screenSize);
+ if (n != 0) return n;
+
+ return 0;
+}
+
+string
+Configuration::ToString() const
+{
+ string s;
+ if (locale.length() > 0) {
+ if (s.length() > 0) {
+ s += "-";
+ }
+ s += locale;
+ }
+ return s;
+}
+
+bool
+split_locale(const string& in, string* language, string* region)
+{
+ const int len = in.length();
+ if (len == 2) {
+ if (isalpha(in[0]) && isalpha(in[1])) {
+ *language = in;
+ region->clear();
+ return true;
+ } else {
+ return false;
+ }
+ }
+ else if (len == 5) {
+ if (isalpha(in[0]) && isalpha(in[1]) && (in[2] == '_' || in[2] == '-')
+ && isalpha(in[3]) && isalpha(in[4])) {
+ language->assign(in.c_str(), 2);
+ region->assign(in.c_str()+3, 2);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ else {
+ return false;
+ }
+}
+
diff --git a/tools/localize/Configuration.h b/tools/localize/Configuration.h
new file mode 100644
index 0000000..f91bf04
--- /dev/null
+++ b/tools/localize/Configuration.h
@@ -0,0 +1,38 @@
+#ifndef CONFIGURATION_H
+#define CONFIGURATION_H
+
+#include <string>
+
+using namespace std;
+
+struct Configuration
+{
+ string locale;
+ string vendor;
+ string orientation;
+ string density;
+ string touchscreen;
+ string keyboard;
+ string navigation;
+ string screenSize;
+
+ // Compare two configurations
+ int Compare(const Configuration& that) const;
+
+ inline bool operator<(const Configuration& that) const { return Compare(that) < 0; }
+ inline bool operator<=(const Configuration& that) const { return Compare(that) <= 0; }
+ inline bool operator==(const Configuration& that) const { return Compare(that) == 0; }
+ inline bool operator!=(const Configuration& that) const { return Compare(that) != 0; }
+ inline bool operator>=(const Configuration& that) const { return Compare(that) >= 0; }
+ inline bool operator>(const Configuration& that) const { return Compare(that) > 0; }
+
+ // Parse a directory name, like "values-en-rUS". Return the first segment in resType.
+ bool ParseDiectoryName(const string& dir, string* resType);
+
+ string ToString() const;
+};
+
+bool split_locale(const string& in, string* language, string* region);
+
+
+#endif // CONFIGURATION_H
diff --git a/tools/localize/Perforce.cpp b/tools/localize/Perforce.cpp
new file mode 100644
index 0000000..3425668
--- /dev/null
+++ b/tools/localize/Perforce.cpp
@@ -0,0 +1,230 @@
+#include "Perforce.h"
+#include "log.h"
+#include <string.h>
+#include <stdlib.h>
+#include <sstream>
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/wait.h>
+
+using namespace std;
+
+extern char** environ;
+
+int
+Perforce::RunCommand(const string& cmd, string* result, bool printOnFailure)
+{
+ int err;
+ int outPipe[2];
+ int errPipe[2];
+ pid_t pid;
+
+ log_printf("Perforce::RunCommand: %s\n", cmd.c_str());
+
+ err = pipe(outPipe);
+ err |= pipe(errPipe);
+ if (err == -1) {
+ printf("couldn't create pipe. exiting.\n");
+ exit(1);
+ return -1;
+ }
+
+ pid = fork();
+ if (pid == -1) {
+ printf("couldn't fork. eixiting\n");
+ exit(1);
+ return -1;
+ }
+ else if (pid == 0) {
+ char const* args[] = {
+ "/bin/sh",
+ "-c",
+ cmd.c_str(),
+ NULL
+ };
+ close(outPipe[0]);
+ close(errPipe[0]);
+ dup2(outPipe[1], 1);
+ dup2(errPipe[1], 2);
+ execve(args[0], (char* const*)args, environ);
+ // done
+ }
+
+ close(outPipe[1]);
+ close(errPipe[1]);
+
+ result->clear();
+
+ char buf[1024];
+
+ // stdout
+ while (true) {
+ size_t amt = read(outPipe[0], buf, sizeof(buf));
+ result->append(buf, amt);
+ if (amt <= 0) {
+ break;
+ }
+ }
+
+ // stderr -- the messages are short so it ought to just fit in the buffer
+ string error;
+ while (true) {
+ size_t amt = read(errPipe[0], buf, sizeof(buf));
+ error.append(buf, amt);
+ if (amt <= 0) {
+ break;
+ }
+ }
+
+ close(outPipe[0]);
+ close(errPipe[0]);
+
+ waitpid(pid, &err, 0);
+ if (WIFEXITED(err)) {
+ err = WEXITSTATUS(err);
+ } else {
+ err = -1;
+ }
+ if (err != 0 && printOnFailure) {
+ write(2, error.c_str(), error.length());
+ }
+ return err;
+}
+
+int
+Perforce::GetResourceFileNames(const string& version, const string& base,
+ const vector<string>& apps, vector<string>* results,
+ bool printOnFailure)
+{
+ int err;
+ string text;
+ stringstream cmd;
+
+ cmd << "p4 files";
+
+ const size_t I = apps.size();
+ for (size_t i=0; i<I; i++) {
+ cmd << " \"" << base << '/' << apps[i] << "/res/values/strings.xml@" << version << '"';
+ }
+
+ err = RunCommand(cmd.str(), &text, printOnFailure);
+
+ const char* str = text.c_str();
+ while (*str) {
+ const char* lineend = strchr(str, '\n');
+ if (lineend == str) {
+ str++;
+ continue;
+ }
+ if (lineend-str > 1023) {
+ fprintf(stderr, "line too long!\n");
+ return 1;
+ }
+
+ string s(str, lineend-str);
+
+ char filename[1024];
+ char edit[1024];
+ int count = sscanf(str, "%[^#]#%*d - %s change %*d %*[^\n]\n", filename, edit);
+
+ if (count == 2 && 0 != strcmp("delete", edit)) {
+ results->push_back(string(filename));
+ }
+
+ str = lineend + 1;
+ }
+
+ return err;
+}
+
+int
+Perforce::GetFile(const string& file, const string& version, string* result,
+ bool printOnFailure)
+{
+ stringstream cmd;
+ cmd << "p4 print -q \"" << file << '@' << version << '"';
+ return RunCommand(cmd.str(), result, printOnFailure);
+}
+
+string
+Perforce::GetCurrentChange(bool printOnFailure)
+{
+ int err;
+ string text;
+
+ err = RunCommand("p4 changes -m 1 \\#have", &text, printOnFailure);
+ if (err != 0) {
+ return "";
+ }
+
+ long long n;
+ int count = sscanf(text.c_str(), "Change %lld on", &n);
+ if (count != 1) {
+ return "";
+ }
+
+ char result[100];
+ sprintf(result, "%lld", n);
+
+ return string(result);
+}
+
+static int
+do_files(const string& op, const vector<string>& files, bool printOnFailure)
+{
+ string text;
+ stringstream cmd;
+
+ cmd << "p4 " << op;
+
+ const size_t I = files.size();
+ for (size_t i=0; i<I; i++) {
+ cmd << " \"" << files[i] << "\"";
+ }
+
+ return Perforce::RunCommand(cmd.str(), &text, printOnFailure);
+}
+
+int
+Perforce::EditFiles(const vector<string>& files, bool printOnFailure)
+{
+ return do_files("edit", files, printOnFailure);
+}
+
+int
+Perforce::AddFiles(const vector<string>& files, bool printOnFailure)
+{
+ return do_files("add", files, printOnFailure);
+}
+
+int
+Perforce::DeleteFiles(const vector<string>& files, bool printOnFailure)
+{
+ return do_files("delete", files, printOnFailure);
+}
+
+string
+Perforce::Where(const string& depotPath, bool printOnFailure)
+{
+ int err;
+ string text;
+ string cmd = "p4 where ";
+ cmd += depotPath;
+
+ err = RunCommand(cmd, &text, printOnFailure);
+ if (err != 0) {
+ return "";
+ }
+
+ size_t index = text.find(' ');
+ if (index == text.npos) {
+ return "";
+ }
+ index = text.find(' ', index+1)+1;
+ if (index == text.npos) {
+ return "";
+ }
+
+ return text.substr(index, text.length()-index-1);
+}
+
diff --git a/tools/localize/Perforce.h b/tools/localize/Perforce.h
new file mode 100644
index 0000000..522797d
--- /dev/null
+++ b/tools/localize/Perforce.h
@@ -0,0 +1,25 @@
+#ifndef PERFORCE_H
+#define PERFORCE_H
+
+#include <string>
+#include <vector>
+
+using namespace std;
+
+class Perforce
+{
+public:
+ static int RunCommand(const string& cmd, string* result, bool printOnFailure);
+ static int GetResourceFileNames(const string& version, const string& base,
+ const vector<string>& apps, vector<string>* result,
+ bool printOnFailure);
+ static int GetFile(const string& file, const string& version, string* result,
+ bool printOnFailure);
+ static string GetCurrentChange(bool printOnFailure);
+ static int EditFiles(const vector<string>& filename, bool printOnFailure);
+ static int AddFiles(const vector<string>& files, bool printOnFailure);
+ static int DeleteFiles(const vector<string>& files, bool printOnFailure);
+ static string Where(const string& depotPath, bool printOnFailure);
+};
+
+#endif // PERFORCE_H
diff --git a/tools/localize/Perforce_test.cpp b/tools/localize/Perforce_test.cpp
new file mode 100644
index 0000000..142b20e
--- /dev/null
+++ b/tools/localize/Perforce_test.cpp
@@ -0,0 +1,62 @@
+#include "Perforce.h"
+#include <stdio.h>
+
+static int
+RunCommand_test()
+{
+ string result;
+ int err = Perforce::RunCommand("p4 help csommands", &result, true);
+ printf("err=%d result=[[%s]]\n", err, result.c_str());
+ return 0;
+}
+
+static int
+GetResourceFileNames_test()
+{
+ vector<string> results;
+ vector<string> apps;
+ apps.push_back("apps/common");
+ apps.push_back("apps/Contacts");
+ int err = Perforce::GetResourceFileNames("43019", "//device", apps, &results, true);
+ if (err != 0) {
+ return err;
+ }
+ if (results.size() != 2) {
+ return 1;
+ }
+ if (results[0] != "//device/apps/common/res/values/strings.xml") {
+ return 1;
+ }
+ if (results[1] != "//device/apps/Contacts/res/values/strings.xml") {
+ return 1;
+ }
+ if (false) {
+ for (size_t i=0; i<results.size(); i++) {
+ printf("[%zd] '%s'\n", i, results[i].c_str());
+ }
+ }
+ return 0;
+}
+
+static int
+GetFile_test()
+{
+ string result;
+ int err = Perforce::GetFile("//device/Makefile", "296", &result, true);
+ printf("err=%d result=[[%s]]\n", err, result.c_str());
+ return 0;
+}
+
+int
+Perforce_test()
+{
+ bool all = false;
+ int err = 0;
+
+ if (all) err |= RunCommand_test();
+ if (all) err |= GetResourceFileNames_test();
+ if (all) err |= GetFile_test();
+
+ return err;
+}
+
diff --git a/tools/localize/SourcePos.cpp b/tools/localize/SourcePos.cpp
new file mode 100644
index 0000000..9d7c5c6
--- /dev/null
+++ b/tools/localize/SourcePos.cpp
@@ -0,0 +1,166 @@
+#include "SourcePos.h"
+
+#include <stdarg.h>
+#include <set>
+
+using namespace std;
+
+const SourcePos GENERATED_POS("<generated>", -1);
+
+// ErrorPos
+// =============================================================================
+struct ErrorPos
+{
+ string file;
+ int line;
+ string error;
+
+ ErrorPos();
+ ErrorPos(const ErrorPos& that);
+ ErrorPos(const string& file, int line, const string& error);
+ ~ErrorPos();
+ bool operator<(const ErrorPos& rhs) const;
+ bool operator==(const ErrorPos& rhs) const;
+ ErrorPos& operator=(const ErrorPos& rhs);
+
+ void Print(FILE* to) const;
+};
+
+static set<ErrorPos> g_errors;
+
+ErrorPos::ErrorPos()
+{
+}
+
+ErrorPos::ErrorPos(const ErrorPos& that)
+ :file(that.file),
+ line(that.line),
+ error(that.error)
+{
+}
+
+ErrorPos::ErrorPos(const string& f, int l, const string& e)
+ :file(f),
+ line(l),
+ error(e)
+{
+}
+
+ErrorPos::~ErrorPos()
+{
+}
+
+bool
+ErrorPos::operator<(const ErrorPos& rhs) const
+{
+ if (this->file < rhs.file) return true;
+ if (this->file == rhs.file) {
+ if (this->line < rhs.line) return true;
+ if (this->line == rhs.line) {
+ if (this->error < rhs.error) return true;
+ }
+ }
+ return false;
+}
+
+bool
+ErrorPos::operator==(const ErrorPos& rhs) const
+{
+ return this->file == rhs.file
+ && this->line == rhs.line
+ && this->error == rhs.error;
+}
+
+ErrorPos&
+ErrorPos::operator=(const ErrorPos& rhs)
+{
+ this->file = rhs.file;
+ this->line = rhs.line;
+ this->error = rhs.error;
+ return *this;
+}
+
+void
+ErrorPos::Print(FILE* to) const
+{
+ if (this->line >= 0) {
+ fprintf(to, "%s:%d: %s\n", this->file.c_str(), this->line, this->error.c_str());
+ } else {
+ fprintf(to, "%s: %s\n", this->file.c_str(), this->error.c_str());
+ }
+}
+
+// SourcePos
+// =============================================================================
+SourcePos::SourcePos(const string& f, int l)
+ : file(f), line(l)
+{
+}
+
+SourcePos::SourcePos(const SourcePos& that)
+ : file(that.file), line(that.line)
+{
+}
+
+SourcePos::SourcePos()
+ : file("???", 0)
+{
+}
+
+SourcePos::~SourcePos()
+{
+}
+
+string
+SourcePos::ToString() const
+{
+ char buf[1024];
+ if (this->line >= 0) {
+ snprintf(buf, sizeof(buf)-1, "%s:%d", this->file.c_str(), this->line);
+ } else {
+ snprintf(buf, sizeof(buf)-1, "%s:", this->file.c_str());
+ }
+ buf[sizeof(buf)-1] = '\0';
+ return string(buf);
+}
+
+int
+SourcePos::Error(const char* fmt, ...) const
+{
+ int retval=0;
+ char buf[1024];
+ va_list ap;
+ va_start(ap, fmt);
+ retval = vsnprintf(buf, sizeof(buf), fmt, ap);
+ va_end(ap);
+ char* p = buf + retval - 1;
+ while (p > buf && *p == '\n') {
+ *p = '\0';
+ p--;
+ }
+ ErrorPos err(this->file, this->line, string(buf));
+ if (g_errors.find(err) == g_errors.end()) {
+ err.Print(stderr);
+ g_errors.insert(err);
+ }
+ return retval;
+}
+
+bool
+SourcePos::HasErrors()
+{
+ return g_errors.size() > 0;
+}
+
+void
+SourcePos::PrintErrors(FILE* to)
+{
+ set<ErrorPos>::const_iterator it;
+ for (it=g_errors.begin(); it!=g_errors.end(); it++) {
+ it->Print(to);
+ }
+}
+
+
+
+
diff --git a/tools/localize/SourcePos.h b/tools/localize/SourcePos.h
new file mode 100644
index 0000000..5027129
--- /dev/null
+++ b/tools/localize/SourcePos.h
@@ -0,0 +1,28 @@
+#ifndef SOURCEPOS_H
+#define SOURCEPOS_H
+
+#include <string>
+
+using namespace std;
+
+class SourcePos
+{
+public:
+ string file;
+ int line;
+
+ SourcePos(const string& f, int l);
+ SourcePos(const SourcePos& that);
+ SourcePos();
+ ~SourcePos();
+
+ string ToString() const;
+ int Error(const char* fmt, ...) const;
+
+ static bool HasErrors();
+ static void PrintErrors(FILE* to);
+};
+
+extern const SourcePos GENERATED_POS;
+
+#endif // SOURCEPOS_H
diff --git a/tools/localize/Values.cpp b/tools/localize/Values.cpp
new file mode 100644
index 0000000..e396f8b
--- /dev/null
+++ b/tools/localize/Values.cpp
@@ -0,0 +1,134 @@
+#include "Values.h"
+#include <stdlib.h>
+
+
+// =====================================================================================
+StringResource::StringResource(const SourcePos& p, const string& f, const Configuration& c,
+ const string& i, int ix, XMLNode* v, const int ve, const string& vs,
+ const string& cmnt)
+ :pos(p),
+ file(f),
+ config(c),
+ id(i),
+ index(ix),
+ value(v),
+ version(ve),
+ versionString(vs),
+ comment(cmnt)
+{
+}
+
+StringResource::StringResource()
+ :pos(),
+ file(),
+ config(),
+ id(),
+ index(-1),
+ value(NULL),
+ version(),
+ versionString(),
+ comment()
+{
+}
+
+StringResource::StringResource(const StringResource& that)
+ :pos(that.pos),
+ file(that.file),
+ config(that.config),
+ id(that.id),
+ index(that.index),
+ value(that.value),
+ version(that.version),
+ versionString(that.versionString),
+ comment(that.comment)
+{
+}
+
+int
+StringResource::Compare(const StringResource& that) const
+{
+ if (file != that.file) {
+ return file < that.file ? -1 : 1;
+ }
+ if (id != that.id) {
+ return id < that.id ? -1 : 1;
+ }
+ if (index != that.index) {
+ return index - that.index;
+ }
+ if (config != that.config) {
+ return config < that.config ? -1 : 1;
+ }
+ if (version != that.version) {
+ return version < that.version ? -1 : 1;
+ }
+ return 0;
+}
+
+string
+StringResource::TypedID() const
+{
+ string result;
+ if (index < 0) {
+ result = "string:";
+ } else {
+ char n[20];
+ sprintf(n, "%d:", index);
+ result = "array:";
+ result += n;
+ }
+ result += id;
+ return result;
+}
+
+static void
+split(const string& raw, vector<string>*parts)
+{
+ size_t index = 0;
+ while (true) {
+ size_t next = raw.find(':', index);
+ if (next != raw.npos) {
+ parts->push_back(string(raw, index, next-index));
+ index = next + 1;
+ } else {
+ parts->push_back(string(raw, index));
+ break;
+ }
+ }
+}
+
+bool
+StringResource::ParseTypedID(const string& raw, string* id, int* index)
+{
+ vector<string> parts;
+ split(raw, &parts);
+
+ const size_t N = parts.size();
+
+ for (size_t i=0; i<N; i++) {
+ if (parts[i].length() == 0) {
+ return false;
+ }
+ }
+
+ if (N == 2 && parts[0] == "string") {
+ *id = parts[1];
+ *index = -1;
+ return true;
+ }
+ else if (N == 3 && parts[0] == "array") {
+ char* p;
+ int n = (int)strtol(parts[1].c_str(), &p, 0);
+ if (*p == '\0') {
+ *id = parts[2];
+ *index = n;
+ return true;
+ } else {
+ return false;
+ }
+ }
+ else {
+ return false;
+ }
+}
+
diff --git a/tools/localize/Values.h b/tools/localize/Values.h
new file mode 100644
index 0000000..0a60b6d
--- /dev/null
+++ b/tools/localize/Values.h
@@ -0,0 +1,48 @@
+#ifndef VALUES_H
+#define VALUES_H
+
+#include "Configuration.h"
+#include "XMLHandler.h"
+
+#include <string>
+
+using namespace std;
+
+enum {
+ CURRENT_VERSION,
+ OLD_VERSION
+};
+
+struct StringResource
+{
+ StringResource();
+ StringResource(const SourcePos& pos, const string& file, const Configuration& config,
+ const string& id, int index, XMLNode* value,
+ int version, const string& versionString, const string& comment = "");
+ StringResource(const StringResource& that);
+
+ // Compare two configurations
+ int Compare(const StringResource& that) const;
+
+ inline bool operator<(const StringResource& that) const { return Compare(that) < 0; }
+ inline bool operator<=(const StringResource& that) const { return Compare(that) <= 0; }
+ inline bool operator==(const StringResource& that) const { return Compare(that) == 0; }
+ inline bool operator!=(const StringResource& that) const { return Compare(that) != 0; }
+ inline bool operator>=(const StringResource& that) const { return Compare(that) >= 0; }
+ inline bool operator>(const StringResource& that) const { return Compare(that) > 0; }
+
+ string TypedID() const;
+ static bool ParseTypedID(const string& typed, string* id, int* index);
+
+ SourcePos pos;
+ string file;
+ Configuration config;
+ string id;
+ int index;
+ XMLNode* value;
+ int version;
+ string versionString;
+ string comment;
+};
+
+#endif // VALUES_H
diff --git a/tools/localize/ValuesFile.cpp b/tools/localize/ValuesFile.cpp
new file mode 100644
index 0000000..bd6f494
--- /dev/null
+++ b/tools/localize/ValuesFile.cpp
@@ -0,0 +1,266 @@
+#include "ValuesFile.h"
+
+#include "XMLHandler.h"
+
+#include <algorithm>
+#include <fcntl.h>
+#include <expat.h>
+#include <unistd.h>
+#include <errno.h>
+
+using namespace std;
+
+const char* const ANDROID_XMLNS = "http://schemas.android.com/apk/res/android";
+const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2";
+
+const char *const NS_MAP[] = {
+ "android", ANDROID_XMLNS,
+ "xliff", XLIFF_XMLNS,
+ NULL, NULL
+};
+
+const XMLNamespaceMap ANDROID_NAMESPACES(NS_MAP);
+
+
+// =====================================================================================
+class ArrayHandler : public XMLHandler
+{
+public:
+ ArrayHandler(ValuesFile* vf, int version, const string& versionString, const string& id);
+
+ virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next);
+ virtual int OnText(const SourcePos& pos, const string& text);
+ virtual int OnComment(const SourcePos& pos, const string& text);
+
+private:
+ ValuesFile* m_vf;
+ int m_version;
+ int m_index;
+ string m_versionString;
+ string m_id;
+ string m_comment;
+};
+
+ArrayHandler::ArrayHandler(ValuesFile* vf, int version, const string& versionString,
+ const string& id)
+ :m_vf(vf),
+ m_version(version),
+ m_index(0),
+ m_versionString(versionString),
+ m_id(id)
+{
+}
+
+int
+ArrayHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next)
+{
+ if (ns == "" && name == "item") {
+ XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT);
+ m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(),
+ m_id, m_index, node, m_version, m_versionString,
+ trim_string(m_comment)));
+ *next = new NodeHandler(node, XMLNode::EXACT);
+ m_index++;
+ m_comment = "";
+ return 0;
+ } else {
+ pos.Error("invalid <%s> element inside <array>\n", name.c_str());
+ return 1;
+ }
+}
+
+int
+ArrayHandler::OnText(const SourcePos& pos, const string& text)
+{
+ return 0;
+}
+
+int
+ArrayHandler::OnComment(const SourcePos& pos, const string& text)
+{
+ m_comment += text;
+ return 0;
+}
+
+// =====================================================================================
+class ValuesHandler : public XMLHandler
+{
+public:
+ ValuesHandler(ValuesFile* vf, int version, const string& versionString);
+
+ virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next);
+ virtual int OnText(const SourcePos& pos, const string& text);
+ virtual int OnComment(const SourcePos& pos, const string& text);
+
+private:
+ ValuesFile* m_vf;
+ int m_version;
+ string m_versionString;
+ string m_comment;
+};
+
+ValuesHandler::ValuesHandler(ValuesFile* vf, int version, const string& versionString)
+ :m_vf(vf),
+ m_version(version),
+ m_versionString(versionString)
+{
+}
+
+int
+ValuesHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next)
+{
+ if (ns == "" && name == "string") {
+ string id = XMLAttribute::Find(attrs, "", "name", "");
+ XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT);
+ m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(),
+ id, -1, node, m_version, m_versionString,
+ trim_string(m_comment)));
+ *next = new NodeHandler(node, XMLNode::EXACT);
+ }
+ else if (ns == "" && name == "array") {
+ string id = XMLAttribute::Find(attrs, "", "name", "");
+ *next = new ArrayHandler(m_vf, m_version, m_versionString, id);
+ }
+ m_comment = "";
+ return 0;
+}
+
+int
+ValuesHandler::OnText(const SourcePos& pos, const string& text)
+{
+ return 0;
+}
+
+int
+ValuesHandler::OnComment(const SourcePos& pos, const string& text)
+{
+ m_comment += text;
+ return 0;
+}
+
+// =====================================================================================
+ValuesFile::ValuesFile(const Configuration& config)
+ :m_config(config),
+ m_strings(),
+ m_arrays()
+{
+}
+
+ValuesFile::~ValuesFile()
+{
+}
+
+ValuesFile*
+ValuesFile::ParseFile(const string& filename, const Configuration& config,
+ int version, const string& versionString)
+{
+ ValuesFile* result = new ValuesFile(config);
+
+ TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString));
+ XMLHandler::ParseFile(filename, &top);
+
+ return result;
+}
+
+ValuesFile*
+ValuesFile::ParseString(const string& filename, const string& text, const Configuration& config,
+ int version, const string& versionString)
+{
+ ValuesFile* result = new ValuesFile(config);
+
+ TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString));
+ XMLHandler::ParseString(filename, text, &top);
+
+ return result;
+}
+
+const Configuration&
+ValuesFile::GetConfiguration() const
+{
+ return m_config;
+}
+
+void
+ValuesFile::AddString(const StringResource& str)
+{
+ if (str.index < 0) {
+ m_strings.insert(str);
+ } else {
+ m_arrays[str.id].insert(str);
+ }
+}
+
+set<StringResource>
+ValuesFile::GetStrings() const
+{
+ set<StringResource> result = m_strings;
+
+ for (map<string,set<StringResource> >::const_iterator it = m_arrays.begin();
+ it != m_arrays.end(); it++) {
+ result.insert(it->second.begin(), it->second.end());
+ }
+
+ return result;
+}
+
+XMLNode*
+ValuesFile::ToXMLNode() const
+{
+ XMLNode* root;
+
+ // <resources>
+ {
+ vector<XMLAttribute> attrs;
+ ANDROID_NAMESPACES.AddToAttributes(&attrs);
+ root = XMLNode::NewElement(GENERATED_POS, "", "resources", attrs, XMLNode::PRETTY);
+ }
+
+ // <array>
+ for (map<string,set<StringResource> >::const_iterator it = m_arrays.begin();
+ it != m_arrays.end(); it++) {
+ vector<XMLAttribute> arrayAttrs;
+ arrayAttrs.push_back(XMLAttribute("", "name", it->first));
+ const set<StringResource>& items = it->second;
+ XMLNode* arrayNode = XMLNode::NewElement(items.begin()->pos, "", "array", arrayAttrs,
+ XMLNode::PRETTY);
+ root->EditChildren().push_back(arrayNode);
+
+ // <item>
+ for (set<StringResource>::const_iterator item = items.begin();
+ item != items.end(); item++) {
+ XMLNode* itemNode = item->value->Clone();
+ itemNode->SetName("", "item");
+ itemNode->EditAttributes().clear();
+ arrayNode->EditChildren().push_back(itemNode);
+ }
+ }
+
+ // <string>
+ for (set<StringResource>::const_iterator it=m_strings.begin(); it!=m_strings.end(); it++) {
+ const StringResource& str = *it;
+ vector<XMLAttribute> attrs;
+ XMLNode* strNode = str.value->Clone();
+ strNode->SetName("", "string");
+ strNode->EditAttributes().clear();
+ strNode->EditAttributes().push_back(XMLAttribute("", "name", str.id));
+ root->EditChildren().push_back(strNode);
+ }
+
+ return root;
+}
+
+string
+ValuesFile::ToString() const
+{
+ XMLNode* xml = ToXMLNode();
+ string s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
+ s += xml->ToString(ANDROID_NAMESPACES);
+ delete xml;
+ s += '\n';
+ return s;
+}
+
diff --git a/tools/localize/ValuesFile.h b/tools/localize/ValuesFile.h
new file mode 100644
index 0000000..752fd78
--- /dev/null
+++ b/tools/localize/ValuesFile.h
@@ -0,0 +1,52 @@
+#ifndef VALUES_FILE_H
+#define VALUES_FILE_H
+
+#include "SourcePos.h"
+#include "Configuration.h"
+#include "XMLHandler.h"
+#include "Values.h"
+
+#include <string>
+#include <set>
+
+using namespace std;
+
+extern const XMLNamespaceMap ANDROID_NAMESPACES;
+
+class ValuesFile
+{
+public:
+ ValuesFile(const Configuration& config);
+
+ static ValuesFile* ParseFile(const string& filename, const Configuration& config,
+ int version, const string& versionString);
+ static ValuesFile* ParseString(const string& filename, const string& text,
+ const Configuration& config,
+ int version, const string& versionString);
+ ~ValuesFile();
+
+ const Configuration& GetConfiguration() const;
+
+ void AddString(const StringResource& str);
+ set<StringResource> GetStrings() const;
+
+ // exports this file as a n XMLNode, you own this object
+ XMLNode* ToXMLNode() const;
+
+ // writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of
+ // ToXMLNode()).
+ string ToString() const;
+
+private:
+ class ParseState;
+ friend class ValuesFile::ParseState;
+ friend class StringHandler;
+
+ ValuesFile();
+
+ Configuration m_config;
+ set<StringResource> m_strings;
+ map<string,set<StringResource> > m_arrays;
+};
+
+#endif // VALUES_FILE_H
diff --git a/tools/localize/ValuesFile_test.cpp b/tools/localize/ValuesFile_test.cpp
new file mode 100644
index 0000000..56d2ec2
--- /dev/null
+++ b/tools/localize/ValuesFile_test.cpp
@@ -0,0 +1,54 @@
+#include "ValuesFile.h"
+#include <stdio.h>
+
+int
+ValuesFile_test()
+{
+ int err = 0;
+ Configuration config;
+ config.locale = "zz_ZZ";
+ ValuesFile* vf = ValuesFile::ParseFile("testdata/values/strings.xml", config,
+ OLD_VERSION, "1");
+
+ const set<StringResource>& strings = vf->GetStrings();
+ string canonical = vf->ToString();
+
+ if (false) {
+ printf("Strings (%zd)\n", strings.size());
+ for (set<StringResource>::const_iterator it=strings.begin();
+ it!=strings.end(); it++) {
+ const StringResource& str = *it;
+ printf("%s: '%s'[%d]='%s' (%s) <!-- %s -->\n", str.pos.ToString().c_str(),
+ str.id.c_str(), str.index,
+ str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
+ str.config.ToString().c_str(), str.comment.c_str());
+ }
+
+ printf("XML:[[%s]]\n", canonical.c_str());
+ }
+
+ const char * const EXPECTED =
+ "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"
+ "<resources xmlns:android=\"http://schemas.android.com/apk/res/android\"\n"
+ " xmlns:xliff=\"urn:oasis:names:tc:xliff:document:1.2\">\n"
+ " <array name=\"emailAddressTypes\">\n"
+ " <item>Email</item>\n"
+ " <item>Home</item>\n"
+ " <item>Work</item>\n"
+ " <item>Other\\u2026</item>\n"
+ " </array>\n"
+ " <string name=\"test1\">Discard</string>\n"
+ " <string name=\"test2\">a<b>b<i>c</i></b>d</string>\n"
+ " <string name=\"test3\">a<xliff:g a=\"b\" xliff:a=\"asdf\">bBb</xliff:g>C</string>\n"
+ "</resources>\n";
+
+ if (canonical != EXPECTED) {
+ fprintf(stderr, "ValuesFile_test failed\n");
+ fprintf(stderr, "canonical=[[%s]]\n", canonical.c_str());
+ fprintf(stderr, "EXPECTED=[[%s]]\n", EXPECTED);
+ err = 1;
+ }
+
+ delete vf;
+ return err;
+}
diff --git a/tools/localize/XLIFFFile.cpp b/tools/localize/XLIFFFile.cpp
new file mode 100644
index 0000000..51f81de
--- /dev/null
+++ b/tools/localize/XLIFFFile.cpp
@@ -0,0 +1,609 @@
+#include "XLIFFFile.h"
+
+#include <algorithm>
+#include <sys/time.h>
+#include <time.h>
+
+const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2";
+
+const char *const NS_MAP[] = {
+ "", XLIFF_XMLNS,
+ "xml", XMLNS_XMLNS,
+ NULL, NULL
+};
+
+const XMLNamespaceMap XLIFF_NAMESPACES(NS_MAP);
+
+int
+XLIFFFile::File::Compare(const XLIFFFile::File& that) const
+{
+ if (filename != that.filename) {
+ return filename < that.filename ? -1 : 1;
+ }
+ return 0;
+}
+
+// =====================================================================================
+XLIFFFile::XLIFFFile()
+{
+}
+
+XLIFFFile::~XLIFFFile()
+{
+}
+
+static XMLNode*
+get_unique_node(const XMLNode* parent, const string& ns, const string& name, bool required)
+{
+ size_t count = parent->CountElementsByName(ns, name);
+ if (count == 1) {
+ return parent->GetElementByNameAt(ns, name, 0);
+ } else {
+ if (required) {
+ SourcePos pos = count == 0
+ ? parent->Position()
+ : parent->GetElementByNameAt(XLIFF_XMLNS, name, 1)->Position();
+ pos.Error("<%s> elements must contain exactly one <%s> element",
+ parent->Name().c_str(), name.c_str());
+ }
+ return NULL;
+ }
+}
+
+XLIFFFile*
+XLIFFFile::Parse(const string& filename)
+{
+ XLIFFFile* result = new XLIFFFile();
+
+ XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY);
+ if (root == NULL) {
+ return NULL;
+ }
+
+ // <file>
+ vector<XMLNode*> files = root->GetElementsByName(XLIFF_XMLNS, "file");
+ for (size_t i=0; i<files.size(); i++) {
+ XMLNode* file = files[i];
+
+ string datatype = file->GetAttribute("", "datatype", "");
+ string originalFile = file->GetAttribute("", "original", "");
+
+ Configuration sourceConfig;
+ sourceConfig.locale = file->GetAttribute("", "source-language", "");
+ result->m_sourceConfig = sourceConfig;
+
+ Configuration targetConfig;
+ targetConfig.locale = file->GetAttribute("", "target-language", "");
+ result->m_targetConfig = targetConfig;
+
+ result->m_currentVersion = file->GetAttribute("", "build-num", "");
+ result->m_oldVersion = "old";
+
+ // <body>
+ XMLNode* body = get_unique_node(file, XLIFF_XMLNS, "body", true);
+ if (body == NULL) continue;
+
+ // <trans-unit>
+ vector<XMLNode*> transUnits = body->GetElementsByName(XLIFF_XMLNS, "trans-unit");
+ for (size_t j=0; j<transUnits.size(); j++) {
+ XMLNode* transUnit = transUnits[j];
+
+ string rawID = transUnit->GetAttribute("", "id", "");
+ if (rawID == "") {
+ transUnit->Position().Error("<trans-unit> tag requires an id");
+ continue;
+ }
+ string id;
+ int index;
+
+ if (!StringResource::ParseTypedID(rawID, &id, &index)) {
+ transUnit->Position().Error("<trans-unit> has invalid id '%s'\n", rawID.c_str());
+ continue;
+ }
+
+ // <source>
+ XMLNode* source = get_unique_node(transUnit, XLIFF_XMLNS, "source", false);
+ if (source != NULL) {
+ XMLNode* node = source->Clone();
+ node->SetPrettyRecursive(XMLNode::EXACT);
+ result->AddStringResource(StringResource(source->Position(), originalFile,
+ sourceConfig, id, index, node, CURRENT_VERSION,
+ result->m_currentVersion));
+ }
+
+ // <target>
+ XMLNode* target = get_unique_node(transUnit, XLIFF_XMLNS, "target", false);
+ if (target != NULL) {
+ XMLNode* node = target->Clone();
+ node->SetPrettyRecursive(XMLNode::EXACT);
+ result->AddStringResource(StringResource(target->Position(), originalFile,
+ targetConfig, id, index, node, CURRENT_VERSION,
+ result->m_currentVersion));
+ }
+
+ // <alt-trans>
+ XMLNode* altTrans = get_unique_node(transUnit, XLIFF_XMLNS, "alt-trans", false);
+ if (altTrans != NULL) {
+ // <source>
+ XMLNode* altSource = get_unique_node(altTrans, XLIFF_XMLNS, "source", false);
+ if (altSource != NULL) {
+ XMLNode* node = altSource->Clone();
+ node->SetPrettyRecursive(XMLNode::EXACT);
+ result->AddStringResource(StringResource(altSource->Position(),
+ originalFile, sourceConfig, id, index, node, OLD_VERSION,
+ result->m_oldVersion));
+ }
+
+ // <target>
+ XMLNode* altTarget = get_unique_node(altTrans, XLIFF_XMLNS, "target", false);
+ if (altTarget != NULL) {
+ XMLNode* node = altTarget->Clone();
+ node->SetPrettyRecursive(XMLNode::EXACT);
+ result->AddStringResource(StringResource(altTarget->Position(),
+ originalFile, targetConfig, id, index, node, OLD_VERSION,
+ result->m_oldVersion));
+ }
+ }
+ }
+ }
+ delete root;
+ return result;
+}
+
+XLIFFFile*
+XLIFFFile::Create(const Configuration& sourceConfig, const Configuration& targetConfig,
+ const string& currentVersion)
+{
+ XLIFFFile* result = new XLIFFFile();
+ result->m_sourceConfig = sourceConfig;
+ result->m_targetConfig = targetConfig;
+ result->m_currentVersion = currentVersion;
+ return result;
+}
+
+set<string>
+XLIFFFile::Files() const
+{
+ set<string> result;
+ for (vector<File>::const_iterator f = m_files.begin(); f != m_files.end(); f++) {
+ result.insert(f->filename);
+ }
+ return result;
+}
+
+void
+XLIFFFile::AddStringResource(const StringResource& str)
+{
+ string id = str.TypedID();
+
+ File* f = NULL;
+ const size_t I = m_files.size();
+ for (size_t i=0; i<I; i++) {
+ if (m_files[i].filename == str.file) {
+ f = &m_files[i];
+ break;
+ }
+ }
+ if (f == NULL) {
+ File file;
+ file.filename = str.file;
+ m_files.push_back(file);
+ f = &m_files[I];
+ }
+
+ const size_t J = f->transUnits.size();
+ TransUnit* g = NULL;
+ for (size_t j=0; j<J; j++) {
+ if (f->transUnits[j].id == id) {
+ g = &f->transUnits[j];
+ }
+ }
+ if (g == NULL) {
+ TransUnit group;
+ group.id = id;
+ f->transUnits.push_back(group);
+ g = &f->transUnits[J];
+ }
+
+ StringResource* res = find_string_res(*g, str);
+ if (res == NULL) {
+ return ;
+ }
+ if (res->id != "") {
+ str.pos.Error("Duplicate string resource: %s", res->id.c_str());
+ res->pos.Error("Previous definition here");
+ return ;
+ }
+ *res = str;
+
+ m_strings.insert(str);
+}
+
+void
+XLIFFFile::Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie)
+{
+ const size_t I = m_files.size();
+ for (size_t ix=0, i=I-1; ix<I; ix++, i--) {
+ File& file = m_files[i];
+
+ const size_t J = file.transUnits.size();
+ for (size_t jx=0, j=J-1; jx<J; jx++, j--) {
+ TransUnit& tu = file.transUnits[j];
+
+ bool keep = func(file.filename, tu, cookie);
+ if (!keep) {
+ if (tu.source.id != "") {
+ m_strings.erase(tu.source);
+ }
+ if (tu.target.id != "") {
+ m_strings.erase(tu.target);
+ }
+ if (tu.altSource.id != "") {
+ m_strings.erase(tu.altSource);
+ }
+ if (tu.altTarget.id != "") {
+ m_strings.erase(tu.altTarget);
+ }
+ file.transUnits.erase(file.transUnits.begin()+j);
+ }
+ }
+ if (file.transUnits.size() == 0) {
+ m_files.erase(m_files.begin()+i);
+ }
+ }
+}
+
+void
+XLIFFFile::Map(void (*func)(const string&,TransUnit*,void*), void* cookie)
+{
+ const size_t I = m_files.size();
+ for (size_t i=0; i<I; i++) {
+ File& file = m_files[i];
+
+ const size_t J = file.transUnits.size();
+ for (size_t j=0; j<J; j++) {
+ func(file.filename, &(file.transUnits[j]), cookie);
+ }
+ }
+}
+
+TransUnit*
+XLIFFFile::EditTransUnit(const string& filename, const string& id)
+{
+ const size_t I = m_files.size();
+ for (size_t ix=0, i=I-1; ix<I; ix++, i--) {
+ File& file = m_files[i];
+ if (file.filename == filename) {
+ const size_t J = file.transUnits.size();
+ for (size_t jx=0, j=J-1; jx<J; jx++, j--) {
+ TransUnit& tu = file.transUnits[j];
+ if (tu.id == id) {
+ return &tu;
+ }
+ }
+ }
+ }
+ return NULL;
+}
+
+StringResource*
+XLIFFFile::find_string_res(TransUnit& g, const StringResource& str)
+{
+ int index;
+ if (str.version == CURRENT_VERSION) {
+ index = 0;
+ }
+ else if (str.version == OLD_VERSION) {
+ index = 2;
+ }
+ else {
+ str.pos.Error("Internal Error %s:%d\n", __FILE__, __LINE__);
+ return NULL;
+ }
+ if (str.config == m_sourceConfig) {
+ // index += 0;
+ }
+ else if (str.config == m_targetConfig) {
+ index += 1;
+ }
+ else {
+ str.pos.Error("unknown config for string %s: %s", str.id.c_str(),
+ str.config.ToString().c_str());
+ return NULL;
+ }
+ switch (index) {
+ case 0:
+ return &g.source;
+ case 1:
+ return &g.target;
+ case 2:
+ return &g.altSource;
+ case 3:
+ return &g.altTarget;
+ }
+ str.pos.Error("Internal Error %s:%d\n", __FILE__, __LINE__);
+ return NULL;
+}
+
+int
+convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID)
+{
+ int err = 0;
+ if (original->Type() == XMLNode::TEXT) {
+ addTo->EditChildren().push_back(original->Clone());
+ return 0;
+ } else {
+ string ctype;
+ if (original->Namespace() == "") {
+ if (original->Name() == "b") {
+ ctype = "bold";
+ }
+ else if (original->Name() == "i") {
+ ctype = "italic";
+ }
+ else if (original->Name() == "u") {
+ ctype = "underline";
+ }
+ }
+ if (ctype != "") {
+ vector<XMLAttribute> attrs;
+ attrs.push_back(XMLAttribute(XLIFF_XMLNS, "ctype", ctype));
+ XMLNode* copy = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, "g",
+ attrs, XMLNode::EXACT);
+
+ const vector<XMLNode*>& children = original->Children();
+ size_t I = children.size();
+ for (size_t i=0; i<I; i++) {
+ err |= convert_html_to_xliff(children[i], name, copy, phID);
+ }
+ return err;
+ }
+ else {
+ if (original->Namespace() == XLIFF_XMLNS) {
+ addTo->EditChildren().push_back(original->Clone());
+ return 0;
+ } else {
+ if (original->Namespace() == "") {
+ // flatten out the tag into ph tags -- but only if there is no namespace
+ // that's still unsupported because propagating the xmlns attribute is hard.
+ vector<XMLAttribute> attrs;
+ char idStr[30];
+ (*phID)++;
+ sprintf(idStr, "id-%d", *phID);
+ attrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", idStr));
+
+ if (original->Children().size() == 0) {
+ XMLNode* ph = XMLNode::NewElement(original->Position(), XLIFF_XMLNS,
+ "ph", attrs, XMLNode::EXACT);
+ ph->EditChildren().push_back(
+ XMLNode::NewText(original->Position(),
+ original->ToString(XLIFF_NAMESPACES),
+ XMLNode::EXACT));
+ addTo->EditChildren().push_back(ph);
+ } else {
+ XMLNode* begin = XMLNode::NewElement(original->Position(), XLIFF_XMLNS,
+ "bpt", attrs, XMLNode::EXACT);
+ begin->EditChildren().push_back(
+ XMLNode::NewText(original->Position(),
+ original->OpenTagToString(XLIFF_NAMESPACES, XMLNode::EXACT),
+ XMLNode::EXACT));
+ XMLNode* end = XMLNode::NewElement(original->Position(), XLIFF_XMLNS,
+ "ept", attrs, XMLNode::EXACT);
+ string endText = "</";
+ endText += original->Name();
+ endText += ">";
+ end->EditChildren().push_back(XMLNode::NewText(original->Position(),
+ endText, XMLNode::EXACT));
+
+ addTo->EditChildren().push_back(begin);
+
+ const vector<XMLNode*>& children = original->Children();
+ size_t I = children.size();
+ for (size_t i=0; i<I; i++) {
+ err |= convert_html_to_xliff(children[i], name, addTo, phID);
+ }
+
+ addTo->EditChildren().push_back(end);
+ }
+ return err;
+ } else {
+ original->Position().Error("invalid <%s> element in <%s> tag\n",
+ original->Name().c_str(), name.c_str());
+ return 1;
+ }
+ }
+ }
+ }
+}
+
+XMLNode*
+create_string_node(const StringResource& str, const string& name)
+{
+ vector<XMLAttribute> attrs;
+ attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve"));
+ XMLNode* node = XMLNode::NewElement(str.pos, XLIFF_XMLNS, name, attrs, XMLNode::EXACT);
+
+ const vector<XMLNode*>& children = str.value->Children();
+ size_t I = children.size();
+ int err = 0;
+ for (size_t i=0; i<I; i++) {
+ int phID = 0;
+ err |= convert_html_to_xliff(children[i], name, node, &phID);
+ }
+
+ if (err != 0) {
+ delete node;
+ }
+ return node;
+}
+
+static bool
+compare_id(const TransUnit& lhs, const TransUnit& rhs)
+{
+ string lid, rid;
+ int lindex, rindex;
+ StringResource::ParseTypedID(lhs.id, &lid, &lindex);
+ StringResource::ParseTypedID(rhs.id, &rid, &rindex);
+ if (lid < rid) return true;
+ if (lid == rid && lindex < rindex) return true;
+ return false;
+}
+
+XMLNode*
+XLIFFFile::ToXMLNode() const
+{
+ XMLNode* root;
+ size_t N;
+
+ // <xliff>
+ {
+ vector<XMLAttribute> attrs;
+ XLIFF_NAMESPACES.AddToAttributes(&attrs);
+ attrs.push_back(XMLAttribute(XLIFF_XMLNS, "version", "1.2"));
+ root = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "xliff", attrs, XMLNode::PRETTY);
+ }
+
+ vector<TransUnit> groups;
+
+ // <file>
+ vector<File> files = m_files;
+ sort(files.begin(), files.end());
+ const size_t I = files.size();
+ for (size_t i=0; i<I; i++) {
+ const File& file = files[i];
+
+ vector<XMLAttribute> fileAttrs;
+ fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "datatype", "x-android-res"));
+ fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "original", file.filename));
+
+ struct timeval tv;
+ struct timezone tz;
+ gettimeofday(&tv, &tz);
+ fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "date", trim_string(ctime(&tv.tv_sec))));
+
+ fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "source-language", m_sourceConfig.locale));
+ fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "target-language", m_targetConfig.locale));
+ fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "build-num", m_currentVersion));
+
+ XMLNode* fileNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "file", fileAttrs,
+ XMLNode::PRETTY);
+ root->EditChildren().push_back(fileNode);
+
+ // <body>
+ XMLNode* bodyNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "body",
+ vector<XMLAttribute>(), XMLNode::PRETTY);
+ fileNode->EditChildren().push_back(bodyNode);
+
+ // <trans-unit>
+ vector<TransUnit> transUnits = file.transUnits;
+ sort(transUnits.begin(), transUnits.end(), compare_id);
+ const size_t J = transUnits.size();
+ for (size_t j=0; j<J; j++) {
+ const TransUnit& transUnit = transUnits[j];
+
+ vector<XMLAttribute> tuAttrs;
+
+ // strings start with string:
+ tuAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", transUnit.id));
+ XMLNode* transUnitNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "trans-unit",
+ tuAttrs, XMLNode::PRETTY);
+ bodyNode->EditChildren().push_back(transUnitNode);
+
+ // <extradata>
+ if (transUnit.source.comment != "") {
+ vector<XMLAttribute> extradataAttrs;
+ XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "extradata",
+ extradataAttrs, XMLNode::EXACT);
+ transUnitNode->EditChildren().push_back(extraNode);
+ extraNode->EditChildren().push_back(
+ XMLNode::NewText(GENERATED_POS, transUnit.source.comment,
+ XMLNode::PRETTY));
+ }
+
+ // <source>
+ if (transUnit.source.id != "") {
+ transUnitNode->EditChildren().push_back(
+ create_string_node(transUnit.source, "source"));
+ }
+
+ // <target>
+ if (transUnit.target.id != "") {
+ transUnitNode->EditChildren().push_back(
+ create_string_node(transUnit.target, "target"));
+ }
+
+ // <alt-trans>
+ if (transUnit.altSource.id != "" || transUnit.altTarget.id != ""
+ || transUnit.rejectComment != "") {
+ vector<XMLAttribute> altTransAttrs;
+ XMLNode* altTransNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "alt-trans",
+ altTransAttrs, XMLNode::PRETTY);
+ transUnitNode->EditChildren().push_back(altTransNode);
+
+ // <extradata>
+ if (transUnit.rejectComment != "") {
+ vector<XMLAttribute> extradataAttrs;
+ XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS,
+ "extradata", extradataAttrs,
+ XMLNode::EXACT);
+ altTransNode->EditChildren().push_back(extraNode);
+ extraNode->EditChildren().push_back(
+ XMLNode::NewText(GENERATED_POS, transUnit.rejectComment,
+ XMLNode::PRETTY));
+ }
+
+ // <source>
+ if (transUnit.altSource.id != "") {
+ altTransNode->EditChildren().push_back(
+ create_string_node(transUnit.altSource, "source"));
+ }
+
+ // <target>
+ if (transUnit.altTarget.id != "") {
+ altTransNode->EditChildren().push_back(
+ create_string_node(transUnit.altTarget, "target"));
+ }
+ }
+
+ }
+ }
+
+ return root;
+}
+
+
+string
+XLIFFFile::ToString() const
+{
+ XMLNode* xml = ToXMLNode();
+ string s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n";
+ s += xml->ToString(XLIFF_NAMESPACES);
+ delete xml;
+ s += '\n';
+ return s;
+}
+
+Stats
+XLIFFFile::GetStats(const string& config) const
+{
+ Stats stat;
+ stat.config = config;
+ stat.files = m_files.size();
+ stat.toBeTranslated = 0;
+ stat.noComments = 0;
+
+ for (vector<File>::const_iterator file=m_files.begin(); file!=m_files.end(); file++) {
+ stat.toBeTranslated += file->transUnits.size();
+
+ for (vector<TransUnit>::const_iterator tu=file->transUnits.begin();
+ tu!=file->transUnits.end(); tu++) {
+ if (tu->source.comment == "") {
+ stat.noComments++;
+ }
+ }
+ }
+
+ stat.totalStrings = stat.toBeTranslated;
+
+ return stat;
+}
diff --git a/tools/localize/XLIFFFile.h b/tools/localize/XLIFFFile.h
new file mode 100644
index 0000000..a93d479
--- /dev/null
+++ b/tools/localize/XLIFFFile.h
@@ -0,0 +1,98 @@
+#ifndef XLIFF_FILE_H
+#define XLIFF_FILE_H
+
+#include "Values.h"
+
+#include "Configuration.h"
+
+#include <set>
+
+using namespace std;
+
+extern const XMLNamespaceMap XLIFF_NAMESPACES;
+
+extern const char*const XLIFF_XMLNS;
+
+struct Stats
+{
+ string config;
+ size_t files;
+ size_t toBeTranslated;
+ size_t noComments;
+ size_t totalStrings;
+};
+
+struct TransUnit {
+ string id;
+ StringResource source;
+ StringResource target;
+ StringResource altSource;
+ StringResource altTarget;
+ string rejectComment;
+};
+
+class XLIFFFile
+{
+public:
+ static XLIFFFile* Parse(const string& filename);
+ static XLIFFFile* Create(const Configuration& sourceConfig, const Configuration& targetConfig,
+ const string& currentVersion);
+ ~XLIFFFile();
+
+ inline const Configuration& SourceConfig() const { return m_sourceConfig; }
+ inline const Configuration& TargetConfig() const { return m_targetConfig; }
+
+ inline const string& CurrentVersion() const { return m_currentVersion; }
+ inline const string& OldVersion() const { return m_oldVersion; }
+
+ set<string> Files() const;
+
+ void AddStringResource(const StringResource& res);
+ inline set<StringResource> const& GetStringResources() const { return m_strings; }
+ bool FindStringResource(const string& filename, int version, bool source);
+
+ void Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie);
+ void Map(void (*func)(const string&,TransUnit*,void*), void* cookie);
+
+ TransUnit* EditTransUnit(const string& file, const string& id);
+
+ // exports this file as a n XMLNode, you own this object
+ XMLNode* ToXMLNode() const;
+
+ // writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of
+ // ToXMLNode()).
+ string ToString() const;
+
+ Stats GetStats(const string& config) const;
+
+private:
+ struct File {
+ int Compare(const File& that) const;
+
+ inline bool operator<(const File& that) const { return Compare(that) < 0; }
+ inline bool operator<=(const File& that) const { return Compare(that) <= 0; }
+ inline bool operator==(const File& that) const { return Compare(that) == 0; }
+ inline bool operator!=(const File& that) const { return Compare(that) != 0; }
+ inline bool operator>=(const File& that) const { return Compare(that) >= 0; }
+ inline bool operator>(const File& that) const { return Compare(that) > 0; }
+
+ string filename;
+ vector<TransUnit> transUnits;
+ };
+
+ XLIFFFile();
+ StringResource* find_string_res(TransUnit& g, const StringResource& str);
+
+ Configuration m_sourceConfig;
+ Configuration m_targetConfig;
+
+ string m_currentVersion;
+ string m_oldVersion;
+
+ set<StringResource> m_strings;
+ vector<File> m_files;
+};
+
+int convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID);
+
+#endif // XLIFF_FILE_H
diff --git a/tools/localize/XLIFFFile_test.cpp b/tools/localize/XLIFFFile_test.cpp
new file mode 100644
index 0000000..52ed4c3
--- /dev/null
+++ b/tools/localize/XLIFFFile_test.cpp
@@ -0,0 +1,115 @@
+#include "XLIFFFile.h"
+#include <stdio.h>
+#include "ValuesFile.h"
+
+XMLNode* create_string_node(const StringResource& str, const string& name);
+
+static int
+Parse_test()
+{
+ XLIFFFile* xf = XLIFFFile::Parse("testdata/xliff1.xliff");
+ if (xf == NULL) {
+ return 1;
+ }
+
+ set<StringResource> const& strings = xf->GetStringResources();
+
+ if (false) {
+ for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
+ const StringResource& str = *it;
+ printf("STRING!!! id=%s index=%d value='%s' pos=%s file=%s version=%d(%s)\n",
+ str.id.c_str(), str.index,
+ str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
+ str.pos.ToString().c_str(), str.file.c_str(), str.version,
+ str.versionString.c_str());
+ }
+ printf("XML:[[%s]]\n", xf->ToString().c_str());
+ }
+
+ delete xf;
+ return 0;
+}
+
+static XMLNode*
+add_html_tag(XMLNode* addTo, const string& tag)
+{
+ vector<XMLAttribute> attrs;
+ XMLNode* node = XMLNode::NewElement(GENERATED_POS, "", tag, attrs, XMLNode::EXACT);
+ addTo->EditChildren().push_back(node);
+ return node;
+}
+
+static int
+create_string_node_test()
+{
+ int err = 0;
+ StringResource res;
+ vector<XMLAttribute> attrs;
+ res.value = XMLNode::NewElement(GENERATED_POS, "", "something", attrs, XMLNode::EXACT);
+ res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " begin ", XMLNode::EXACT));
+
+ XMLNode* child;
+
+ child = add_html_tag(res.value, "b");
+ child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT));
+
+ child = add_html_tag(res.value, "i");
+ child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "i", XMLNode::EXACT));
+
+ child = add_html_tag(child, "b");
+ child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT));
+
+ child = add_html_tag(res.value, "u");
+ child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "u", XMLNode::EXACT));
+
+
+ res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " end ", XMLNode::EXACT));
+
+ XMLNode* xliff = create_string_node(res, "blah");
+
+ string oldString = res.value->ToString(XLIFF_NAMESPACES);
+ string newString = xliff->ToString(XLIFF_NAMESPACES);
+
+ if (false) {
+ printf("OLD=\"%s\"\n", oldString.c_str());
+ printf("NEW=\"%s\"\n", newString.c_str());
+ }
+
+ const char* const EXPECTED_OLD
+ = "<something> begin <b>b</b><i>i<b>b</b></i><u>u</u> end </something>";
+ if (oldString != EXPECTED_OLD) {
+ fprintf(stderr, "oldString mismatch:\n");
+ fprintf(stderr, " expected='%s'\n", EXPECTED_OLD);
+ fprintf(stderr, " actual='%s'\n", oldString.c_str());
+ err |= 1;
+ }
+
+ const char* const EXPECTED_NEW
+ = "<blah xml:space=\"preserve\"> begin <g ctype=\"bold\">b</g>"
+ "<g ctype=\"italic\">i<g ctype=\"bold\">b</g></g><g ctype=\"underline\">u</g>"
+ " end </blah>";
+ if (newString != EXPECTED_NEW) {
+ fprintf(stderr, "newString mismatch:\n");
+ fprintf(stderr, " expected='%s'\n", EXPECTED_NEW);
+ fprintf(stderr, " actual='%s'\n", newString.c_str());
+ err |= 1;
+ }
+
+ if (err != 0) {
+ fprintf(stderr, "create_string_node_test failed\n");
+ }
+ return err;
+}
+
+int
+XLIFFFile_test()
+{
+ bool all = true;
+ int err = 0;
+
+ if (all) err |= Parse_test();
+ if (all) err |= create_string_node_test();
+
+ return err;
+}
+
diff --git a/tools/localize/XMLHandler.cpp b/tools/localize/XMLHandler.cpp
new file mode 100644
index 0000000..3fab211
--- /dev/null
+++ b/tools/localize/XMLHandler.cpp
@@ -0,0 +1,793 @@
+#include "XMLHandler.h"
+
+#include <algorithm>
+#include <expat.h>
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <errno.h>
+
+#define NS_SEPARATOR 1
+#define MORE_INDENT " "
+
+static string
+xml_text_escape(const string& s)
+{
+ string result;
+ const size_t N = s.length();
+ for (size_t i=0; i<N; i++) {
+ char c = s[i];
+ switch (c) {
+ case '<':
+ result += "&lt;";
+ break;
+ case '>':
+ result += "&gt;";
+ break;
+ case '&':
+ result += "&amp;";
+ break;
+ default:
+ result += c;
+ break;
+ }
+ }
+ return result;
+}
+
+static string
+xml_attr_escape(const string& s)
+{
+ string result;
+ const size_t N = s.length();
+ for (size_t i=0; i<N; i++) {
+ char c = s[i];
+ switch (c) {
+ case '\"':
+ result += "&quot;";
+ break;
+ default:
+ result += c;
+ break;
+ }
+ }
+ return result;
+}
+
+XMLNamespaceMap::XMLNamespaceMap()
+{
+}
+
+XMLNamespaceMap::XMLNamespaceMap(char const*const* nspaces)
+
+{
+ while (*nspaces) {
+ m_map[nspaces[1]] = nspaces[0];
+ nspaces += 2;
+ }
+}
+
+string
+XMLNamespaceMap::Get(const string& ns) const
+{
+ if (ns == "xml") {
+ return ns;
+ }
+ map<string,string>::const_iterator it = m_map.find(ns);
+ if (it == m_map.end()) {
+ return "";
+ } else {
+ return it->second;
+ }
+}
+
+string
+XMLNamespaceMap::GetPrefix(const string& ns) const
+{
+ if (ns == "") {
+ return "";
+ }
+ map<string,string>::const_iterator it = m_map.find(ns);
+ if (it != m_map.end()) {
+ if (it->second == "") {
+ return "";
+ } else {
+ return it->second + ":";
+ }
+ } else {
+ return ":"; // invalid
+ }
+}
+
+void
+XMLNamespaceMap::AddToAttributes(vector<XMLAttribute>* attrs) const
+{
+ map<string,string>::const_iterator it;
+ for (it=m_map.begin(); it!=m_map.end(); it++) {
+ if (it->second == "xml") {
+ continue;
+ }
+ XMLAttribute attr;
+ if (it->second == "") {
+ attr.name = "xmlns";
+ } else {
+ attr.name = "xmlns:";
+ attr.name += it->second;
+ }
+ attr.value = it->first;
+ attrs->push_back(attr);
+ }
+}
+
+XMLAttribute::XMLAttribute()
+{
+}
+
+XMLAttribute::XMLAttribute(const XMLAttribute& that)
+ :ns(that.ns),
+ name(that.name),
+ value(that.value)
+{
+}
+
+XMLAttribute::XMLAttribute(string n, string na, string v)
+ :ns(n),
+ name(na),
+ value(v)
+{
+}
+
+XMLAttribute::~XMLAttribute()
+{
+}
+
+int
+XMLAttribute::Compare(const XMLAttribute& that) const
+{
+ if (ns != that.ns) {
+ return ns < that.ns ? -1 : 1;
+ }
+ if (name != that.name) {
+ return name < that.name ? -1 : 1;
+ }
+ return 0;
+}
+
+string
+XMLAttribute::Find(const vector<XMLAttribute>& list, const string& ns, const string& name,
+ const string& def)
+{
+ const size_t N = list.size();
+ for (size_t i=0; i<N; i++) {
+ const XMLAttribute& attr = list[i];
+ if (attr.ns == ns && attr.name == name) {
+ return attr.value;
+ }
+ }
+ return def;
+}
+
+struct xml_handler_data {
+ vector<XMLHandler*> stack;
+ XML_Parser parser;
+ vector<vector<XMLAttribute>*> attributes;
+ string filename;
+};
+
+XMLNode::XMLNode()
+{
+}
+
+XMLNode::~XMLNode()
+{
+// for_each(m_children.begin(), m_children.end(), delete_object<XMLNode>);
+}
+
+XMLNode*
+XMLNode::Clone() const
+{
+ switch (m_type) {
+ case ELEMENT: {
+ XMLNode* e = XMLNode::NewElement(m_pos, m_ns, m_name, m_attrs, m_pretty);
+ const size_t N = m_children.size();
+ for (size_t i=0; i<N; i++) {
+ e->m_children.push_back(m_children[i]->Clone());
+ }
+ return e;
+ }
+ case TEXT: {
+ return XMLNode::NewText(m_pos, m_text, m_pretty);
+ }
+ default:
+ return NULL;
+ }
+}
+
+XMLNode*
+XMLNode::NewElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, int pretty)
+{
+ XMLNode* node = new XMLNode();
+ node->m_type = ELEMENT;
+ node->m_pretty = pretty;
+ node->m_pos = pos;
+ node->m_ns = ns;
+ node->m_name = name;
+ node->m_attrs = attrs;
+ return node;
+}
+
+XMLNode*
+XMLNode::NewText(const SourcePos& pos, const string& text, int pretty)
+{
+ XMLNode* node = new XMLNode();
+ node->m_type = TEXT;
+ node->m_pretty = pretty;
+ node->m_pos = pos;
+ node->m_text = text;
+ return node;
+}
+
+void
+XMLNode::SetPrettyRecursive(int value)
+{
+ m_pretty = value;
+ const size_t N = m_children.size();
+ for (size_t i=0; i<N; i++) {
+ m_children[i]->SetPrettyRecursive(value);
+ }
+}
+
+string
+XMLNode::ContentsToString(const XMLNamespaceMap& nspaces) const
+{
+ return contents_to_string(nspaces, "");
+}
+
+string
+XMLNode::ToString(const XMLNamespaceMap& nspaces) const
+{
+ return to_string(nspaces, "");
+}
+
+string
+XMLNode::OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const
+{
+ return open_tag_to_string(nspaces, "", pretty);
+}
+
+string
+XMLNode::contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const
+{
+ string result;
+ const size_t N = m_children.size();
+ for (size_t i=0; i<N; i++) {
+ const XMLNode* child = m_children[i];
+ switch (child->Type()) {
+ case ELEMENT:
+ if (m_pretty == PRETTY) {
+ result += '\n';
+ result += indent;
+ }
+ case TEXT:
+ result += child->to_string(nspaces, indent);
+ break;
+ }
+ }
+ return result;
+}
+
+string
+trim_string(const string& str)
+{
+ const char* p = str.c_str();
+ while (*p && isspace(*p)) {
+ p++;
+ }
+ const char* q = str.c_str() + str.length() - 1;
+ while (q > p && isspace(*q)) {
+ q--;
+ }
+ q++;
+ return string(p, q-p);
+}
+
+string
+XMLNode::open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent, int pretty) const
+{
+ if (m_type != ELEMENT) {
+ return "";
+ }
+ string result = "<";
+ result += nspaces.GetPrefix(m_ns);
+ result += m_name;
+
+ vector<XMLAttribute> attrs = m_attrs;
+
+ sort(attrs.begin(), attrs.end());
+
+ const size_t N = attrs.size();
+ for (size_t i=0; i<N; i++) {
+ const XMLAttribute& attr = attrs[i];
+ if (i == 0 || m_pretty == EXACT || pretty == EXACT) {
+ result += ' ';
+ }
+ else {
+ result += "\n";
+ result += indent;
+ result += MORE_INDENT;
+ result += MORE_INDENT;
+ }
+ result += nspaces.GetPrefix(attr.ns);
+ result += attr.name;
+ result += "=\"";
+ result += xml_attr_escape(attr.value);
+ result += '\"';
+ }
+
+ if (m_children.size() > 0) {
+ result += '>';
+ } else {
+ result += " />";
+ }
+ return result;
+}
+
+string
+XMLNode::to_string(const XMLNamespaceMap& nspaces, const string& indent) const
+{
+ switch (m_type)
+ {
+ case TEXT: {
+ if (m_pretty == EXACT) {
+ return xml_text_escape(m_text);
+ } else {
+ return xml_text_escape(trim_string(m_text));
+ }
+ }
+ case ELEMENT: {
+ string result = open_tag_to_string(nspaces, indent, PRETTY);
+
+ if (m_children.size() > 0) {
+ result += contents_to_string(nspaces, indent + MORE_INDENT);
+
+ if (m_pretty == PRETTY && m_children.size() > 0) {
+ result += '\n';
+ result += indent;
+ }
+
+ result += "</";
+ result += nspaces.GetPrefix(m_ns);
+ result += m_name;
+ result += '>';
+ }
+ return result;
+ }
+ default:
+ return "";
+ }
+}
+
+string
+XMLNode::CollapseTextContents() const
+{
+ if (m_type == TEXT) {
+ return m_text;
+ }
+ else if (m_type == ELEMENT) {
+ string result;
+
+ const size_t N=m_children.size();
+ for (size_t i=0; i<N; i++) {
+ result += m_children[i]->CollapseTextContents();
+ }
+
+ return result;
+ }
+ else {
+ return "";
+ }
+}
+
+vector<XMLNode*>
+XMLNode::GetElementsByName(const string& ns, const string& name) const
+{
+ vector<XMLNode*> result;
+ const size_t N=m_children.size();
+ for (size_t i=0; i<N; i++) {
+ XMLNode* child = m_children[i];
+ if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) {
+ result.push_back(child);
+ }
+ }
+ return result;
+}
+
+XMLNode*
+XMLNode::GetElementByNameAt(const string& ns, const string& name, size_t index) const
+{
+ vector<XMLNode*> result;
+ const size_t N=m_children.size();
+ for (size_t i=0; i<N; i++) {
+ XMLNode* child = m_children[i];
+ if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) {
+ if (index == 0) {
+ return child;
+ } else {
+ index--;
+ }
+ }
+ }
+ return NULL;
+}
+
+size_t
+XMLNode::CountElementsByName(const string& ns, const string& name) const
+{
+ size_t result = 0;
+ const size_t N=m_children.size();
+ for (size_t i=0; i<N; i++) {
+ XMLNode* child = m_children[i];
+ if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) {
+ result++;
+ }
+ }
+ return result;
+}
+
+string
+XMLNode::GetAttribute(const string& ns, const string& name, const string& def) const
+{
+ return XMLAttribute::Find(m_attrs, ns, name, def);
+}
+
+static void
+parse_namespace(const char* data, string* ns, string* name)
+{
+ const char* p = strchr(data, NS_SEPARATOR);
+ if (p != NULL) {
+ ns->assign(data, p-data);
+ name->assign(p+1);
+ } else {
+ ns->assign("");
+ name->assign(data);
+ }
+}
+
+static void
+convert_attrs(const char** in, vector<XMLAttribute>* out)
+{
+ while (*in) {
+ XMLAttribute attr;
+ parse_namespace(in[0], &attr.ns, &attr.name);
+ attr.value = in[1];
+ out->push_back(attr);
+ in += 2;
+ }
+}
+
+static bool
+list_contains(const vector<XMLHandler*>& stack, XMLHandler* handler)
+{
+ const size_t N = stack.size();
+ for (size_t i=0; i<N; i++) {
+ if (stack[i] == handler) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static void XMLCALL
+start_element_handler(void *userData, const char *name, const char **attrs)
+{
+ xml_handler_data* data = (xml_handler_data*)userData;
+
+ XMLHandler* handler = data->stack[data->stack.size()-1];
+
+ SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
+ string nsString;
+ string nameString;
+ XMLHandler* next = handler;
+ vector<XMLAttribute> attributes;
+
+ parse_namespace(name, &nsString, &nameString);
+ convert_attrs(attrs, &attributes);
+
+ handler->OnStartElement(pos, nsString, nameString, attributes, &next);
+
+ if (next == NULL) {
+ next = handler;
+ }
+
+ if (next != handler) {
+ next->elementPos = pos;
+ next->elementNamespace = nsString;
+ next->elementName = nameString;
+ next->elementAttributes = attributes;
+ }
+
+ data->stack.push_back(next);
+}
+
+static void XMLCALL
+end_element_handler(void *userData, const char *name)
+{
+ xml_handler_data* data = (xml_handler_data*)userData;
+
+ XMLHandler* handler = data->stack[data->stack.size()-1];
+ data->stack.pop_back();
+
+ SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
+
+ if (!list_contains(data->stack, handler)) {
+ handler->OnDone(pos);
+ if (data->stack.size() > 1) {
+ // not top one
+ delete handler;
+ }
+ }
+
+ handler = data->stack[data->stack.size()-1];
+
+ string nsString;
+ string nameString;
+
+ parse_namespace(name, &nsString, &nameString);
+
+ handler->OnEndElement(pos, nsString, nameString);
+}
+
+static void XMLCALL
+text_handler(void *userData, const XML_Char *s, int len)
+{
+ xml_handler_data* data = (xml_handler_data*)userData;
+ XMLHandler* handler = data->stack[data->stack.size()-1];
+ SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
+ handler->OnText(pos, string(s, len));
+}
+
+static void XMLCALL
+comment_handler(void *userData, const char *comment)
+{
+ xml_handler_data* data = (xml_handler_data*)userData;
+ XMLHandler* handler = data->stack[data->stack.size()-1];
+ SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser));
+ handler->OnComment(pos, string(comment));
+}
+
+bool
+XMLHandler::ParseFile(const string& filename, XMLHandler* handler)
+{
+ char buf[16384];
+ int fd = open(filename.c_str(), O_RDONLY);
+ if (fd < 0) {
+ SourcePos(filename, -1).Error("Unable to open file for read: %s", strerror(errno));
+ return false;
+ }
+
+ XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR);
+ xml_handler_data state;
+ state.stack.push_back(handler);
+ state.parser = parser;
+ state.filename = filename;
+
+ XML_SetUserData(parser, &state);
+ XML_SetElementHandler(parser, start_element_handler, end_element_handler);
+ XML_SetCharacterDataHandler(parser, text_handler);
+ XML_SetCommentHandler(parser, comment_handler);
+
+ ssize_t len;
+ bool done;
+ do {
+ len = read(fd, buf, sizeof(buf));
+ done = len < (ssize_t)sizeof(buf);
+ if (len < 0) {
+ SourcePos(filename, -1).Error("Error reading file: %s\n", strerror(errno));
+ close(fd);
+ return false;
+ }
+ if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) {
+ SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error(
+ "Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser)));
+ close(fd);
+ return false;
+ }
+ } while (!done);
+
+ XML_ParserFree(parser);
+
+ close(fd);
+
+ return true;
+}
+
+bool
+XMLHandler::ParseString(const string& filename, const string& text, XMLHandler* handler)
+{
+ XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR);
+ xml_handler_data state;
+ state.stack.push_back(handler);
+ state.parser = parser;
+ state.filename = filename;
+
+ XML_SetUserData(parser, &state);
+ XML_SetElementHandler(parser, start_element_handler, end_element_handler);
+ XML_SetCharacterDataHandler(parser, text_handler);
+ XML_SetCommentHandler(parser, comment_handler);
+
+ if (XML_Parse(parser, text.c_str(), text.size(), true) == XML_STATUS_ERROR) {
+ SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error(
+ "Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser)));
+ return false;
+ }
+
+ XML_ParserFree(parser);
+
+ return true;
+}
+
+XMLHandler::XMLHandler()
+{
+}
+
+XMLHandler::~XMLHandler()
+{
+}
+
+int
+XMLHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next)
+{
+ return 0;
+}
+
+int
+XMLHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name)
+{
+ return 0;
+}
+
+int
+XMLHandler::OnText(const SourcePos& pos, const string& text)
+{
+ return 0;
+}
+
+int
+XMLHandler::OnComment(const SourcePos& pos, const string& text)
+{
+ return 0;
+}
+
+int
+XMLHandler::OnDone(const SourcePos& pos)
+{
+ return 0;
+}
+
+TopElementHandler::TopElementHandler(const string& ns, const string& name, XMLHandler* next)
+ :m_ns(ns),
+ m_name(name),
+ m_next(next)
+{
+}
+
+int
+TopElementHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next)
+{
+ *next = m_next;
+ return 0;
+}
+
+int
+TopElementHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name)
+{
+ return 0;
+}
+
+int
+TopElementHandler::OnText(const SourcePos& pos, const string& text)
+{
+ return 0;
+}
+
+int
+TopElementHandler::OnDone(const SourcePos& pos)
+{
+ return 0;
+}
+
+
+NodeHandler::NodeHandler(XMLNode* root, int pretty)
+ :m_root(root),
+ m_pretty(pretty)
+{
+ if (root != NULL) {
+ m_nodes.push_back(root);
+ }
+}
+
+NodeHandler::~NodeHandler()
+{
+}
+
+int
+NodeHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next)
+{
+ int pretty;
+ if (XMLAttribute::Find(attrs, XMLNS_XMLNS, "space", "") == "preserve") {
+ pretty = XMLNode::EXACT;
+ } else {
+ if (m_root == NULL) {
+ pretty = m_pretty;
+ } else {
+ pretty = m_nodes[m_nodes.size()-1]->Pretty();
+ }
+ }
+ XMLNode* n = XMLNode::NewElement(pos, ns, name, attrs, pretty);
+ if (m_root == NULL) {
+ m_root = n;
+ } else {
+ m_nodes[m_nodes.size()-1]->EditChildren().push_back(n);
+ }
+ m_nodes.push_back(n);
+ return 0;
+}
+
+int
+NodeHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name)
+{
+ m_nodes.pop_back();
+ return 0;
+}
+
+int
+NodeHandler::OnText(const SourcePos& pos, const string& text)
+{
+ if (m_root == NULL) {
+ return 1;
+ }
+ XMLNode* n = XMLNode::NewText(pos, text, m_nodes[m_nodes.size()-1]->Pretty());
+ m_nodes[m_nodes.size()-1]->EditChildren().push_back(n);
+ return 0;
+}
+
+int
+NodeHandler::OnComment(const SourcePos& pos, const string& text)
+{
+ return 0;
+}
+
+int
+NodeHandler::OnDone(const SourcePos& pos)
+{
+ return 0;
+}
+
+XMLNode*
+NodeHandler::ParseFile(const string& filename, int pretty)
+{
+ NodeHandler handler(NULL, pretty);
+ if (!XMLHandler::ParseFile(filename, &handler)) {
+ fprintf(stderr, "error parsing file: %s\n", filename.c_str());
+ return NULL;
+ }
+ return handler.Root();
+}
+
+XMLNode*
+NodeHandler::ParseString(const string& filename, const string& text, int pretty)
+{
+ NodeHandler handler(NULL, pretty);
+ if (!XMLHandler::ParseString(filename, text, &handler)) {
+ fprintf(stderr, "error parsing file: %s\n", filename.c_str());
+ return NULL;
+ }
+ return handler.Root();
+}
+
+
diff --git a/tools/localize/XMLHandler.h b/tools/localize/XMLHandler.h
new file mode 100644
index 0000000..1130710
--- /dev/null
+++ b/tools/localize/XMLHandler.h
@@ -0,0 +1,197 @@
+#ifndef XML_H
+#define XML_H
+
+#include "SourcePos.h"
+
+#include <string>
+#include <vector>
+#include <map>
+
+#define XMLNS_XMLNS "http://www.w3.org/XML/1998/namespace"
+
+using namespace std;
+
+string trim_string(const string& str);
+
+struct XMLAttribute
+{
+ string ns;
+ string name;
+ string value;
+
+ XMLAttribute();
+ XMLAttribute(const XMLAttribute& that);
+ XMLAttribute(string ns, string name, string value);
+ ~XMLAttribute();
+
+ int Compare(const XMLAttribute& that) const;
+
+ inline bool operator<(const XMLAttribute& that) const { return Compare(that) < 0; }
+ inline bool operator<=(const XMLAttribute& that) const { return Compare(that) <= 0; }
+ inline bool operator==(const XMLAttribute& that) const { return Compare(that) == 0; }
+ inline bool operator!=(const XMLAttribute& that) const { return Compare(that) != 0; }
+ inline bool operator>=(const XMLAttribute& that) const { return Compare(that) >= 0; }
+ inline bool operator>(const XMLAttribute& that) const { return Compare(that) > 0; }
+
+ static string Find(const vector<XMLAttribute>& list,
+ const string& ns, const string& name, const string& def);
+};
+
+class XMLNamespaceMap
+{
+public:
+ XMLNamespaceMap();
+ XMLNamespaceMap(char const*const* nspaces);
+ string Get(const string& ns) const;
+ string GetPrefix(const string& ns) const;
+ void AddToAttributes(vector<XMLAttribute>* attrs) const;
+private:
+ map<string,string> m_map;
+};
+
+struct XMLNode
+{
+public:
+ enum {
+ EXACT = 0,
+ PRETTY = 1
+ };
+
+ enum {
+ ELEMENT = 0,
+ TEXT = 1
+ };
+
+ static XMLNode* NewElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, int pretty);
+ static XMLNode* NewText(const SourcePos& pos, const string& text, int pretty);
+
+ ~XMLNode();
+
+ // a deep copy
+ XMLNode* Clone() const;
+
+ inline int Type() const { return m_type; }
+ inline int Pretty() const { return m_pretty; }
+ void SetPrettyRecursive(int value);
+ string ContentsToString(const XMLNamespaceMap& nspaces) const;
+ string ToString(const XMLNamespaceMap& nspaces) const;
+ string OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const;
+
+ string CollapseTextContents() const;
+
+ inline const SourcePos& Position() const { return m_pos; }
+
+ // element
+ inline string Namespace() const { return m_ns; }
+ inline string Name() const { return m_name; }
+ inline void SetName(const string& ns, const string& n) { m_ns = ns; m_name = n; }
+ inline const vector<XMLAttribute>& Attributes() const { return m_attrs; }
+ inline vector<XMLAttribute>& EditAttributes() { return m_attrs; }
+ inline const vector<XMLNode*>& Children() const { return m_children; }
+ inline vector<XMLNode*>& EditChildren() { return m_children; }
+ vector<XMLNode*> GetElementsByName(const string& ns, const string& name) const;
+ XMLNode* GetElementByNameAt(const string& ns, const string& name, size_t index) const;
+ size_t CountElementsByName(const string& ns, const string& name) const;
+ string GetAttribute(const string& ns, const string& name, const string& def) const;
+
+ // text
+ inline string Text() const { return m_text; }
+
+private:
+ XMLNode();
+ XMLNode(const XMLNode&);
+
+ string contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const;
+ string to_string(const XMLNamespaceMap& nspaces, const string& indent) const;
+ string open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent,
+ int pretty) const;
+
+ int m_type;
+ int m_pretty;
+ SourcePos m_pos;
+
+ // element
+ string m_ns;
+ string m_name;
+ vector<XMLAttribute> m_attrs;
+ vector<XMLNode*> m_children;
+
+ // text
+ string m_text;
+};
+
+class XMLHandler
+{
+public:
+ // information about the element that started us
+ SourcePos elementPos;
+ string elementNamespace;
+ string elementName;
+ vector<XMLAttribute> elementAttributes;
+
+ XMLHandler();
+ virtual ~XMLHandler();
+
+ XMLHandler* parent;
+
+ virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next);
+ virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name);
+ virtual int OnText(const SourcePos& pos, const string& text);
+ virtual int OnComment(const SourcePos& pos, const string& text);
+ virtual int OnDone(const SourcePos& pos);
+
+ static bool ParseFile(const string& filename, XMLHandler* handler);
+ static bool ParseString(const string& filename, const string& text, XMLHandler* handler);
+};
+
+class TopElementHandler : public XMLHandler
+{
+public:
+ TopElementHandler(const string& ns, const string& name, XMLHandler* next);
+
+ virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next);
+ virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name);
+ virtual int OnText(const SourcePos& pos, const string& text);
+ virtual int OnDone(const SourcePos& endPos);
+
+private:
+ string m_ns;
+ string m_name;
+ XMLHandler* m_next;
+};
+
+class NodeHandler : public XMLHandler
+{
+public:
+ // after it's done, you own everything created and added to root
+ NodeHandler(XMLNode* root, int pretty);
+ ~NodeHandler();
+
+ virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name,
+ const vector<XMLAttribute>& attrs, XMLHandler** next);
+ virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name);
+ virtual int OnText(const SourcePos& pos, const string& text);
+ virtual int OnComment(const SourcePos& pos, const string& text);
+ virtual int OnDone(const SourcePos& endPos);
+
+ inline XMLNode* Root() const { return m_root; }
+
+ static XMLNode* ParseFile(const string& filename, int pretty);
+ static XMLNode* ParseString(const string& filename, const string& text, int pretty);
+
+private:
+ XMLNode* m_root;
+ int m_pretty;
+ vector<XMLNode*> m_nodes;
+};
+
+template <class T>
+static void delete_object(T* obj)
+{
+ delete obj;
+}
+
+#endif // XML_H
diff --git a/tools/localize/XMLHandler_test.cpp b/tools/localize/XMLHandler_test.cpp
new file mode 100644
index 0000000..1c81c0c
--- /dev/null
+++ b/tools/localize/XMLHandler_test.cpp
@@ -0,0 +1,133 @@
+#include "XMLHandler.h"
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+const char *const NS_MAP[] = {
+ "xml", XMLNS_XMLNS,
+ NULL, NULL
+};
+
+const XMLNamespaceMap NO_NAMESPACES(NS_MAP);
+
+char const*const EXPECTED_EXACT =
+ "<ASDF>\n"
+ " <a id=\"system\" old-cl=\"1\" new-cl=\"43019\">\n"
+ " <app dir=\"apps/common\" />\n"
+ " </a>\n"
+ " <a id=\"samples\" old-cl=\"1\" new-cl=\"43019\">asdf\n"
+ " <app dir=\"samples/NotePad\" />\n"
+ " <app dir=\"samples/LunarLander\" />\n"
+ " <something>a<b>,</b>b </something>\n"
+ " <exact xml:space=\"preserve\">a<b>,</b>b </exact>\n"
+ " </a>\n"
+ "</ASDF>\n";
+
+char const*const EXPECTED_PRETTY =
+ "<ASDF>\n"
+ " <a id=\"system\"\n"
+ " old-cl=\"1\"\n"
+ " new-cl=\"43019\">\n"
+ " <app dir=\"apps/common\" />\n"
+ " </a>\n"
+ " <a id=\"samples\"\n"
+ " old-cl=\"1\"\n"
+ " new-cl=\"43019\">asdf\n"
+ " <app dir=\"samples/NotePad\" />\n"
+ " <app dir=\"samples/LunarLander\" />\n"
+ " <something>a\n"
+ " <b>,\n"
+ " </b>b \n"
+ " </something>\n"
+ " <exact xml:space=\"preserve\">a<b>,</b>b </exact>\n"
+ " </a>\n"
+ "</ASDF>\n";
+
+static string
+read_file(const string& filename)
+{
+ char buf[1024];
+ int fd = open(filename.c_str(), O_RDONLY);
+ if (fd < 0) {
+ return "";
+ }
+ string result;
+ while (true) {
+ ssize_t len = read(fd, buf, sizeof(buf)-1);
+ buf[len] = '\0';
+ if (len <= 0) {
+ break;
+ }
+ result.append(buf, len);
+ }
+ close(fd);
+ return result;
+}
+
+static int
+ParseFile_EXACT_test()
+{
+ XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::EXACT);
+ if (root == NULL) {
+ return 1;
+ }
+ string result = root->ToString(NO_NAMESPACES);
+ delete root;
+ //printf("[[%s]]\n", result.c_str());
+ return result == EXPECTED_EXACT;
+}
+
+static int
+ParseFile_PRETTY_test()
+{
+ XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::PRETTY);
+ if (root == NULL) {
+ return 1;
+ }
+ string result = root->ToString(NO_NAMESPACES);
+ delete root;
+ //printf("[[%s]]\n", result.c_str());
+ return result == EXPECTED_PRETTY;
+}
+
+static int
+ParseString_EXACT_test()
+{
+ string text = read_file("testdata/xml.xml");
+ XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::EXACT);
+ if (root == NULL) {
+ return 1;
+ }
+ string result = root->ToString(NO_NAMESPACES);
+ delete root;
+ //printf("[[%s]]\n", result.c_str());
+ return result == EXPECTED_EXACT;
+}
+
+static int
+ParseString_PRETTY_test()
+{
+ string text = read_file("testdata/xml.xml");
+ XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::PRETTY);
+ if (root == NULL) {
+ return 1;
+ }
+ string result = root->ToString(NO_NAMESPACES);
+ delete root;
+ //printf("[[%s]]\n", result.c_str());
+ return result == EXPECTED_PRETTY;
+}
+
+int
+XMLHandler_test()
+{
+ int err = 0;
+ bool all = true;
+
+ if (all) err |= ParseFile_EXACT_test();
+ if (all) err |= ParseFile_PRETTY_test();
+ if (all) err |= ParseString_EXACT_test();
+ if (all) err |= ParseString_PRETTY_test();
+
+ return err;
+}
diff --git a/tools/localize/XMLNode.h b/tools/localize/XMLNode.h
new file mode 100644
index 0000000..bfb9f55
--- /dev/null
+++ b/tools/localize/XMLNode.h
@@ -0,0 +1,19 @@
+#ifndef XMLNODE_H
+#define XMLNODE_H
+
+#include <string>
+
+using namespace std;
+
+struct XMLAttribute
+{
+ string ns;
+ string name;
+ string value;
+
+ static string Find(const vector<XMLAttribute>& list,
+ const string& ns, const string& name, const string& def);
+};
+
+
+#endif // XMLNODE_H
diff --git a/tools/localize/file_utils.cpp b/tools/localize/file_utils.cpp
new file mode 100644
index 0000000..bb82a9c
--- /dev/null
+++ b/tools/localize/file_utils.cpp
@@ -0,0 +1,143 @@
+#include <string.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include "file_utils.h"
+#include "Perforce.h"
+#include <sys/fcntl.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include <host/Directories.h>
+#include "log.h"
+
+string
+translated_file_name(const string& file, const string& locale)
+{
+ const char* str = file.c_str();
+ const char* p = str + file.length();
+ const char* rest = NULL;
+ const char* values = p;
+
+ while (p > str) {
+ p--;
+ if (*p == '/') {
+ rest = values;
+ values = p;
+ if (0 == strncmp("values", values+1, rest-values-1)) {
+ break;
+ }
+ }
+ }
+ values++;
+
+ string result(str, values-str);
+ result.append(values, rest-values);
+
+ string language, region;
+ if (locale == "") {
+ language = "";
+ region = "";
+ }
+ else if (!split_locale(locale, &language, &region)) {
+ return "";
+ }
+
+ if (language != "") {
+ result += '-';
+ result += language;
+ }
+ if (region != "") {
+ result += "-r";
+ result += region;
+ }
+
+ result += rest;
+
+ return result;
+}
+
+ValuesFile*
+get_values_file(const string& filename, const Configuration& configuration,
+ int version, const string& versionString, bool printOnFailure)
+{
+ int err;
+ string text;
+
+ log_printf("get_values_file filename=%s\n", filename.c_str());
+ err = Perforce::GetFile(filename, versionString, &text, printOnFailure);
+ if (err != 0 || text == "") {
+ return NULL;
+ }
+
+ ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version,
+ versionString);
+ if (result == NULL) {
+ fprintf(stderr, "unable to parse file: %s\n", filename.c_str());
+ exit(1);
+ }
+ return result;
+}
+
+ValuesFile*
+get_local_values_file(const string& filename, const Configuration& configuration,
+ int version, const string& versionString, bool printOnFailure)
+{
+ int err;
+ string text;
+ char buf[2049];
+ int fd;
+ ssize_t amt;
+
+ fd = open(filename.c_str(), O_RDONLY);
+ if (fd == -1) {
+ fprintf(stderr, "unable to open file: %s\n", filename.c_str());
+ return NULL;
+ }
+
+ while ((amt = read(fd, buf, sizeof(buf)-1)) > 0) {
+ text.append(buf, amt);
+ }
+
+ close(fd);
+
+ if (text == "") {
+ return NULL;
+ }
+
+ ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version,
+ versionString);
+ if (result == NULL) {
+ fprintf(stderr, "unable to parse file: %s\n", filename.c_str());
+ exit(1);
+ }
+ return result;
+}
+
+void
+print_file_status(size_t j, size_t J, const string& message)
+{
+ printf("\r%s file %zd of %zd...", message.c_str(), j, J);
+ fflush(stdout);
+}
+
+int
+write_to_file(const string& filename, const string& text)
+{
+ mkdirs(parent_dir(filename).c_str());
+ int fd = open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666);
+ if (fd < 0) {
+ fprintf(stderr, "unable to open file for write (%s): %s\n", strerror(errno),
+ filename.c_str());
+ return -1;
+ }
+
+ ssize_t amt = write(fd, text.c_str(), text.length());
+
+ close(fd);
+
+ if (amt < 0) {
+ return amt;
+ }
+ return amt == (ssize_t)text.length() ? 0 : -1;
+}
+
+
diff --git a/tools/localize/file_utils.h b/tools/localize/file_utils.h
new file mode 100644
index 0000000..3b3fa21
--- /dev/null
+++ b/tools/localize/file_utils.h
@@ -0,0 +1,21 @@
+#ifndef FILE_UTILS_H
+#define FILE_UTILS_H
+
+#include "ValuesFile.h"
+#include "Configuration.h"
+#include <string>
+
+using namespace std;
+
+string translated_file_name(const string& file, const string& locale);
+
+ValuesFile* get_values_file(const string& filename, const Configuration& configuration,
+ int version, const string& versionString, bool printOnFailure);
+ValuesFile* get_local_values_file(const string& filename, const Configuration& configuration,
+ int version, const string& versionString, bool printOnFailure);
+
+void print_file_status(size_t j, size_t J, const string& message = "Reading");
+int write_to_file(const string& filename, const string& text);
+
+
+#endif // FILE_UTILS_H
diff --git a/tools/localize/localize.cpp b/tools/localize/localize.cpp
new file mode 100644
index 0000000..c0d84cc
--- /dev/null
+++ b/tools/localize/localize.cpp
@@ -0,0 +1,767 @@
+#include "SourcePos.h"
+#include "ValuesFile.h"
+#include "XLIFFFile.h"
+#include "Perforce.h"
+#include "merge_res_and_xliff.h"
+#include "localize.h"
+#include "file_utils.h"
+#include "res_check.h"
+#include "xmb.h"
+
+#include <host/pseudolocalize.h>
+
+#include <stdlib.h>
+#include <stdarg.h>
+#include <sstream>
+#include <stdio.h>
+#include <string.h>
+
+using namespace std;
+
+FILE* g_logFile = NULL;
+
+int test();
+
+int
+read_settings(const string& filename, map<string,Settings>* result, const string& rootDir)
+{
+ XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY);
+ if (root == NULL) {
+ SourcePos(filename, -1).Error("Error reading file.");
+ return 1;
+ }
+
+ // <configuration>
+ vector<XMLNode*> configNodes = root->GetElementsByName("", "configuration");
+ const size_t I = configNodes.size();
+ for (size_t i=0; i<I; i++) {
+ const XMLNode* configNode = configNodes[i];
+
+ Settings settings;
+ settings.id = configNode->GetAttribute("", "id", "");
+ if (settings.id == "") {
+ configNode->Position().Error("<configuration> needs an id attribute.");
+ delete root;
+ return 1;
+ }
+
+ settings.oldVersion = configNode->GetAttribute("", "old-cl", "");
+
+ settings.currentVersion = configNode->GetAttribute("", "new-cl", "");
+ if (settings.currentVersion == "") {
+ configNode->Position().Error("<configuration> needs a new-cl attribute.");
+ delete root;
+ return 1;
+ }
+
+ // <app>
+ vector<XMLNode*> appNodes = configNode->GetElementsByName("", "app");
+
+ const size_t J = appNodes.size();
+ for (size_t j=0; j<J; j++) {
+ const XMLNode* appNode = appNodes[j];
+
+ string dir = appNode->GetAttribute("", "dir", "");
+ if (dir == "") {
+ appNode->Position().Error("<app> needs a dir attribute.");
+ delete root;
+ return 1;
+ }
+
+ settings.apps.push_back(dir);
+ }
+
+ // <reject>
+ vector<XMLNode*> rejectNodes = configNode->GetElementsByName("", "reject");
+
+ const size_t K = rejectNodes.size();
+ for (size_t k=0; k<K; k++) {
+ const XMLNode* rejectNode = rejectNodes[k];
+
+ Reject reject;
+
+ reject.file = rejectNode->GetAttribute("", "file", "");
+ if (reject.file == "") {
+ rejectNode->Position().Error("<reject> needs a file attribute.");
+ delete root;
+ return 1;
+ }
+ string f = reject.file;
+ reject.file = rootDir;
+ reject.file += '/';
+ reject.file += f;
+
+ reject.name = rejectNode->GetAttribute("", "name", "");
+ if (reject.name == "") {
+ rejectNode->Position().Error("<reject> needs a name attribute.");
+ delete root;
+ return 1;
+ }
+
+ reject.comment = trim_string(rejectNode->CollapseTextContents());
+
+ settings.reject.push_back(reject);
+ }
+
+ (*result)[settings.id] = settings;
+ }
+
+ delete root;
+ return 0;
+}
+
+
+static void
+ValuesFile_to_XLIFFFile(const ValuesFile* values, XLIFFFile* xliff, const string& englishFilename)
+{
+ const set<StringResource>& strings = values->GetStrings();
+ for (set<StringResource>::const_iterator it=strings.begin(); it!=strings.end(); it++) {
+ StringResource res = *it;
+ res.file = englishFilename;
+ xliff->AddStringResource(res);
+ }
+}
+
+static bool
+contains_reject(const Settings& settings, const string& file, const TransUnit& tu)
+{
+ const string name = tu.id;
+ const vector<Reject>& reject = settings.reject;
+ const size_t I = reject.size();
+ for (size_t i=0; i<I; i++) {
+ const Reject& r = reject[i];
+ if (r.file == file && r.name == name) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/**
+ * If it's been rejected, then we keep whatever info we have.
+ *
+ * Implements this truth table:
+ *
+ * S AT AS Keep
+ * -----------------------
+ * 0 0 0 0 (this case can't happen)
+ * 0 0 1 0 (it was there, never translated, and removed)
+ * 0 1 0 0 (somehow it got translated, but it was removed)
+ * 0 1 1 0 (it was removed after having been translated)
+ *
+ * 1 0 0 1 (it was just added)
+ * 1 0 1 1 (it was added, has been changed, but it never got translated)
+ * 1 1 0 1 (somehow it got translated, but we don't know based on what)
+ * 1 1 1 0/1 (it's in both. 0 if S=AS b/c there's no need to retranslate if they're
+ * the same. 1 if S!=AS because S changed, so it should be retranslated)
+ *
+ * The first four are cases where, whatever happened in the past, the string isn't there
+ * now, so it shouldn't be in the XLIFF file.
+ *
+ * For cases 4 and 5, the string has never been translated, so get it translated.
+ *
+ * For case 6, it's unclear where the translated version came from, so we're conservative
+ * and send it back for them to have another shot at.
+ *
+ * For case 7, we have some data. We have two choices. We could rely on the translator's
+ * translation memory or tools to notice that the strings haven't changed, and populate the
+ * <target> field themselves. Or if the string hasn't changed since last time, we can just
+ * not even tell them about it. As the project nears the end, it will be convenient to see
+ * the xliff files reducing in size, so we pick the latter. Obviously, if the string has
+ * changed, then we need to get it retranslated.
+ */
+bool
+keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie)
+{
+ const Settings* settings = reinterpret_cast<const Settings*>(cookie);
+
+ if (contains_reject(*settings, file, unit)) {
+ return true;
+ }
+
+ if (unit.source.id == "") {
+ return false;
+ }
+ if (unit.altTarget.id == "" || unit.altSource.id == "") {
+ return true;
+ }
+ return unit.source.value->ContentsToString(XLIFF_NAMESPACES)
+ != unit.altSource.value->ContentsToString(XLIFF_NAMESPACES);
+}
+
+int
+validate_config(const string& settingsFile, const map<string,Settings>& settings,
+ const string& config)
+{
+ if (settings.find(config) == settings.end()) {
+ SourcePos(settingsFile, -1).Error("settings file does not contain setting: %s\n",
+ config.c_str());
+ return 1;
+ }
+ return 0;
+}
+
+int
+validate_configs(const string& settingsFile, const map<string,Settings>& settings,
+ const vector<string>& configs)
+{
+ int err = 0;
+ for (size_t i=0; i<configs.size(); i++) {
+ string config = configs[i];
+ err |= validate_config(settingsFile, settings, config);
+ }
+ return err;
+}
+
+int
+select_files(vector<string> *resFiles, const string& config,
+ const map<string,Settings>& settings, const string& rootDir)
+{
+ int err;
+ vector<vector<string> > allResFiles;
+ vector<string> configs;
+ configs.push_back(config);
+ err = select_files(&allResFiles, configs, settings, rootDir);
+ if (err == 0) {
+ *resFiles = allResFiles[0];
+ }
+ return err;
+}
+
+int
+select_files(vector<vector<string> > *allResFiles, const vector<string>& configs,
+ const map<string,Settings>& settings, const string& rootDir)
+{
+ int err;
+ printf("Selecting files...");
+ fflush(stdout);
+
+ for (size_t i=0; i<configs.size(); i++) {
+ const string& config = configs[i];
+ const Settings& setting = settings.find(config)->second;
+
+ vector<string> resFiles;
+ err = Perforce::GetResourceFileNames(setting.currentVersion, rootDir,
+ setting.apps, &resFiles, true);
+ if (err != 0) {
+ fprintf(stderr, "error with perforce. bailing\n");
+ return err;
+ }
+
+ allResFiles->push_back(resFiles);
+ }
+ return 0;
+}
+
+static int
+do_export(const string& settingsFile, const string& rootDir, const string& outDir,
+ const string& targetLocale, const vector<string>& configs)
+{
+ bool success = true;
+ int err;
+
+ if (false) {
+ printf("settingsFile=%s\n", settingsFile.c_str());
+ printf("rootDir=%s\n", rootDir.c_str());
+ printf("outDir=%s\n", outDir.c_str());
+ for (size_t i=0; i<configs.size(); i++) {
+ printf("config[%zd]=%s\n", i, configs[i].c_str());
+ }
+ }
+
+ map<string,Settings> settings;
+ err = read_settings(settingsFile, &settings, rootDir);
+ if (err != 0) {
+ return err;
+ }
+
+ err = validate_configs(settingsFile, settings, configs);
+ if (err != 0) {
+ return err;
+ }
+
+ vector<vector<string> > allResFiles;
+ err = select_files(&allResFiles, configs, settings, rootDir);
+ if (err != 0) {
+ return err;
+ }
+
+ size_t totalFileCount = 0;
+ for (size_t i=0; i<allResFiles.size(); i++) {
+ totalFileCount += allResFiles[i].size();
+ }
+ totalFileCount *= 3; // we try all 3 versions of the file
+
+ size_t fileProgress = 0;
+ vector<Stats> stats;
+ vector<pair<string,XLIFFFile*> > xliffs;
+
+ for (size_t i=0; i<configs.size(); i++) {
+ const string& config = configs[i];
+ const Settings& setting = settings[config];
+
+ if (false) {
+ fprintf(stderr, "Configuration: %s (%zd of %zd)\n", config.c_str(), i+1,
+ configs.size());
+ fprintf(stderr, " Old CL: %s\n", setting.oldVersion.c_str());
+ fprintf(stderr, " Current CL: %s\n", setting.currentVersion.c_str());
+ }
+
+ Configuration english;
+ english.locale = "en_US";
+ Configuration translated;
+ translated.locale = targetLocale;
+ XLIFFFile* xliff = XLIFFFile::Create(english, translated, setting.currentVersion);
+
+ const vector<string>& resFiles = allResFiles[i];
+ const size_t J = resFiles.size();
+ for (size_t j=0; j<J; j++) {
+ string resFile = resFiles[j];
+
+ // parse the files into a ValuesFile
+ // pull out the strings and add them to the XLIFFFile
+
+ // current file
+ print_file_status(++fileProgress, totalFileCount);
+ ValuesFile* currentFile = get_values_file(resFile, english, CURRENT_VERSION,
+ setting.currentVersion, true);
+ if (currentFile != NULL) {
+ ValuesFile_to_XLIFFFile(currentFile, xliff, resFile);
+ //printf("currentFile=[%s]\n", currentFile->ToString().c_str());
+ } else {
+ fprintf(stderr, "error reading file %s@%s\n", resFile.c_str(),
+ setting.currentVersion.c_str());
+ success = false;
+ }
+
+ // old file
+ print_file_status(++fileProgress, totalFileCount);
+ ValuesFile* oldFile = get_values_file(resFile, english, OLD_VERSION,
+ setting.oldVersion, false);
+ if (oldFile != NULL) {
+ ValuesFile_to_XLIFFFile(oldFile, xliff, resFile);
+ //printf("oldFile=[%s]\n", oldFile->ToString().c_str());
+ }
+
+ // translated version
+ // (get the head of the tree for the most recent translation, but it's considered
+ // the old one because the "current" one hasn't been made yet, and this goes into
+ // the <alt-trans> tag if necessary
+ print_file_status(++fileProgress, totalFileCount);
+ string transFilename = translated_file_name(resFile, targetLocale);
+ ValuesFile* transFile = get_values_file(transFilename, translated, OLD_VERSION,
+ setting.currentVersion, false);
+ if (transFile != NULL) {
+ ValuesFile_to_XLIFFFile(transFile, xliff, resFile);
+ }
+
+ delete currentFile;
+ delete oldFile;
+ delete transFile;
+ }
+
+ Stats beforeFilterStats = xliff->GetStats(config);
+
+ // run through the XLIFFFile and strip out TransUnits that have identical
+ // old and current source values and are not in the reject list, or just
+ // old values and no source values
+ xliff->Filter(keep_this_trans_unit, (void*)&setting);
+
+ Stats afterFilterStats = xliff->GetStats(config);
+ afterFilterStats.totalStrings = beforeFilterStats.totalStrings;
+
+ // add the reject comments
+ for (vector<Reject>::const_iterator reject = setting.reject.begin();
+ reject != setting.reject.end(); reject++) {
+ TransUnit* tu = xliff->EditTransUnit(reject->file, reject->name);
+ tu->rejectComment = reject->comment;
+ }
+
+ // config-locale-current_cl.xliff
+ stringstream filename;
+ if (outDir != "") {
+ filename << outDir << '/';
+ }
+ filename << config << '-' << targetLocale << '-' << setting.currentVersion << ".xliff";
+ xliffs.push_back(pair<string,XLIFFFile*>(filename.str(), xliff));
+
+ stats.push_back(afterFilterStats);
+ }
+
+ // today is a good day to die
+ if (!success || SourcePos::HasErrors()) {
+ return 1;
+ }
+
+ // write the XLIFF files
+ printf("\nWriting %zd file%s...\n", xliffs.size(), xliffs.size() == 1 ? "" : "s");
+ for (vector<pair<string,XLIFFFile*> >::iterator it = xliffs.begin(); it != xliffs.end(); it++) {
+ const string& filename = it->first;
+ XLIFFFile* xliff = it->second;
+ string text = xliff->ToString();
+ write_to_file(filename, text);
+ }
+
+ // the stats
+ printf("\n"
+ " to without total\n"
+ " config files translate comments strings\n"
+ "-----------------------------------------------------------------------\n");
+ Stats totals;
+ totals.config = "total";
+ totals.files = 0;
+ totals.toBeTranslated = 0;
+ totals.noComments = 0;
+ totals.totalStrings = 0;
+ for (vector<Stats>::iterator it=stats.begin(); it!=stats.end(); it++) {
+ string cfg = it->config;
+ if (cfg.length() > 20) {
+ cfg.resize(20);
+ }
+ printf(" %-20s %-9zd %-9zd %-9zd %-19zd\n", cfg.c_str(), it->files,
+ it->toBeTranslated, it->noComments, it->totalStrings);
+ totals.files += it->files;
+ totals.toBeTranslated += it->toBeTranslated;
+ totals.noComments += it->noComments;
+ totals.totalStrings += it->totalStrings;
+ }
+ if (stats.size() > 1) {
+ printf("-----------------------------------------------------------------------\n"
+ " %-20s %-9zd %-9zd %-9zd %-19zd\n", totals.config.c_str(), totals.files,
+ totals.toBeTranslated, totals.noComments, totals.totalStrings);
+ }
+ printf("\n");
+ return 0;
+}
+
+struct PseudolocalizeSettings {
+ XLIFFFile* xliff;
+ bool expand;
+};
+
+
+string
+pseudolocalize_string(const string& source, const PseudolocalizeSettings* settings)
+{
+ return pseudolocalize_string(source);
+}
+
+static XMLNode*
+pseudolocalize_xml_node(const XMLNode* source, const PseudolocalizeSettings* settings)
+{
+ if (source->Type() == XMLNode::TEXT) {
+ return XMLNode::NewText(source->Position(), pseudolocalize_string(source->Text(), settings),
+ source->Pretty());
+ } else {
+ XMLNode* target;
+ if (source->Namespace() == XLIFF_XMLNS && source->Name() == "g") {
+ // XXX don't translate these
+ target = XMLNode::NewElement(source->Position(), source->Namespace(),
+ source->Name(), source->Attributes(), source->Pretty());
+ } else {
+ target = XMLNode::NewElement(source->Position(), source->Namespace(),
+ source->Name(), source->Attributes(), source->Pretty());
+ }
+
+ const vector<XMLNode*>& children = source->Children();
+ const size_t I = children.size();
+ for (size_t i=0; i<I; i++) {
+ target->EditChildren().push_back(pseudolocalize_xml_node(children[i], settings));
+ }
+
+ return target;
+ }
+}
+
+void
+pseudolocalize_trans_unit(const string&file, TransUnit* unit, void* cookie)
+{
+ const PseudolocalizeSettings* settings = (PseudolocalizeSettings*)cookie;
+
+ const StringResource& source = unit->source;
+ StringResource* target = &unit->target;
+ *target = source;
+
+ target->config = settings->xliff->TargetConfig();
+
+ delete target->value;
+ target->value = pseudolocalize_xml_node(source.value, settings);
+}
+
+int
+pseudolocalize_xliff(XLIFFFile* xliff, bool expand)
+{
+ PseudolocalizeSettings settings;
+
+ settings.xliff = xliff;
+ settings.expand = expand;
+ xliff->Map(pseudolocalize_trans_unit, &settings);
+ return 0;
+}
+
+static int
+do_pseudo(const string& infile, const string& outfile, bool expand)
+{
+ int err;
+
+ XLIFFFile* xliff = XLIFFFile::Parse(infile);
+ if (xliff == NULL) {
+ return 1;
+ }
+
+ pseudolocalize_xliff(xliff, expand);
+
+ err = write_to_file(outfile, xliff->ToString());
+
+ delete xliff;
+
+ return err;
+}
+
+void
+log_printf(const char *fmt, ...)
+{
+ int ret;
+ va_list ap;
+
+ if (g_logFile != NULL) {
+ va_start(ap, fmt);
+ ret = vfprintf(g_logFile, fmt, ap);
+ va_end(ap);
+ fflush(g_logFile);
+ }
+}
+
+void
+close_log_file()
+{
+ if (g_logFile != NULL) {
+ fclose(g_logFile);
+ }
+}
+
+void
+open_log_file(const char* file)
+{
+ g_logFile = fopen(file, "w");
+ printf("log file: %s -- %p\n", file, g_logFile);
+ atexit(close_log_file);
+}
+
+static int
+usage()
+{
+ fprintf(stderr,
+ "usage: localize export OPTIONS CONFIGS...\n"
+ " REQUIRED OPTIONS\n"
+ " --settings SETTINGS The settings file to use. See CONFIGS below.\n"
+ " --root TREE_ROOT The location in Perforce of the files. e.g. //device\n"
+ " --target LOCALE The target locale. See LOCALES below.\n"
+ "\n"
+ " OPTIONAL OPTIONS\n"
+ " --out DIR Directory to put the output files. Defaults to the\n"
+ " current directory if not supplied. Files are\n"
+ " named as follows:\n"
+ " CONFIG-LOCALE-CURRENT_CL.xliff\n"
+ "\n"
+ "\n"
+ "usage: localize import XLIFF_FILE...\n"
+ "\n"
+ "Import a translated XLIFF file back into the tree.\n"
+ "\n"
+ "\n"
+ "usage: localize xlb XMB_FILE VALUES_FILES...\n"
+ "\n"
+ "Read resource files from the tree file and write the corresponding XLB file\n"
+ "\n"
+ "Supply all of the android resource files (values files) to export after that.\n"
+ "\n"
+ "\n"
+ "\n"
+ "CONFIGS\n"
+ "\n"
+ "LOCALES\n"
+ "Locales are specified in the form en_US They will be processed correctly\n"
+ "to locate the resouce files in the tree.\n"
+ "\n"
+ "\n"
+ "usage: localize pseudo OPTIONS INFILE [OUTFILE]\n"
+ " OPTIONAL OPTIONS\n"
+ " --big Pad strings so they get longer.\n"
+ "\n"
+ "Read INFILE, an XLIFF file, and output a pseudotranslated version of that file. If\n"
+ "OUTFILE is specified, the results are written there; otherwise, the results are\n"
+ "written back to INFILE.\n"
+ "\n"
+ "\n"
+ "usage: localize rescheck FILES...\n"
+ "\n"
+ "Reads the base strings and prints warnings about bad resources from the given files.\n"
+ "\n");
+ return 1;
+}
+
+int
+main(int argc, const char** argv)
+{
+ //open_log_file("log.txt");
+ //g_logFile = stdout;
+
+ if (argc == 2 && 0 == strcmp(argv[1], "--test")) {
+ return test();
+ }
+
+ if (argc < 2) {
+ return usage();
+ }
+
+ int index = 1;
+
+ if (0 == strcmp("export", argv[index])) {
+ string settingsFile;
+ string rootDir;
+ string outDir;
+ string baseLocale = "en";
+ string targetLocale;
+ string language, region;
+ vector<string> configs;
+
+ index++;
+ while (index < argc) {
+ if (0 == strcmp("--settings", argv[index])) {
+ settingsFile = argv[index+1];
+ index += 2;
+ }
+ else if (0 == strcmp("--root", argv[index])) {
+ rootDir = argv[index+1];
+ index += 2;
+ }
+ else if (0 == strcmp("--out", argv[index])) {
+ outDir = argv[index+1];
+ index += 2;
+ }
+ else if (0 == strcmp("--target", argv[index])) {
+ targetLocale = argv[index+1];
+ index += 2;
+ }
+ else if (argv[index][0] == '-') {
+ fprintf(stderr, "unknown argument %s\n", argv[index]);
+ return usage();
+ }
+ else {
+ break;
+ }
+ }
+ for (; index<argc; index++) {
+ configs.push_back(argv[index]);
+ }
+
+ if (settingsFile == "" || rootDir == "" || configs.size() == 0 || targetLocale == "") {
+ return usage();
+ }
+ if (!split_locale(targetLocale, &language, &region)) {
+ fprintf(stderr, "illegal --target locale: '%s'\n", targetLocale.c_str());
+ return usage();
+ }
+
+
+ return do_export(settingsFile, rootDir, outDir, targetLocale, configs);
+ }
+ else if (0 == strcmp("import", argv[index])) {
+ vector<string> xliffFilenames;
+
+ index++;
+ for (; index<argc; index++) {
+ xliffFilenames.push_back(argv[index]);
+ }
+
+ return do_merge(xliffFilenames);
+ }
+ else if (0 == strcmp("xlb", argv[index])) {
+ string outfile;
+ vector<string> resFiles;
+
+ index++;
+ if (argc < index+1) {
+ return usage();
+ }
+
+ outfile = argv[index];
+
+ index++;
+ for (; index<argc; index++) {
+ resFiles.push_back(argv[index]);
+ }
+
+ return do_xlb_export(outfile, resFiles);
+ }
+ else if (0 == strcmp("pseudo", argv[index])) {
+ string infile;
+ string outfile;
+ bool big = false;
+
+ index++;
+ while (index < argc) {
+ if (0 == strcmp("--big", argv[index])) {
+ big = true;
+ index += 1;
+ }
+ else if (argv[index][0] == '-') {
+ fprintf(stderr, "unknown argument %s\n", argv[index]);
+ return usage();
+ }
+ else {
+ break;
+ }
+ }
+
+ if (index == argc-1) {
+ infile = argv[index];
+ outfile = argv[index];
+ }
+ else if (index == argc-2) {
+ infile = argv[index];
+ outfile = argv[index+1];
+ }
+ else {
+ fprintf(stderr, "unknown argument %s\n", argv[index]);
+ return usage();
+ }
+
+ return do_pseudo(infile, outfile, big);
+ }
+ else if (0 == strcmp("rescheck", argv[index])) {
+ vector<string> files;
+
+ index++;
+ while (index < argc) {
+ if (argv[index][0] == '-') {
+ fprintf(stderr, "unknown argument %s\n", argv[index]);
+ return usage();
+ }
+ else {
+ break;
+ }
+ }
+ for (; index<argc; index++) {
+ files.push_back(argv[index]);
+ }
+
+ if (files.size() == 0) {
+ return usage();
+ }
+
+ return do_rescheck(files);
+ }
+ else {
+ return usage();
+ }
+
+ if (SourcePos::HasErrors()) {
+ SourcePos::PrintErrors(stderr);
+ return 1;
+ }
+
+ return 0;
+}
+
diff --git a/tools/localize/localize.h b/tools/localize/localize.h
new file mode 100644
index 0000000..615d14e
--- /dev/null
+++ b/tools/localize/localize.h
@@ -0,0 +1,40 @@
+#ifndef LOCALIZE_H
+#define LOCALIZE_H
+
+#include "XLIFFFile.h"
+
+#include <map>
+#include <string>
+
+using namespace std;
+
+struct Reject
+{
+ string file;
+ string name;
+ string comment;
+};
+
+struct Settings
+{
+ string id;
+ string oldVersion;
+ string currentVersion;
+ vector<string> apps;
+ vector<Reject> reject;
+};
+
+int read_settings(const string& filename, map<string,Settings>* result, const string& rootDir);
+string translated_file_name(const string& file, const string& locale);
+bool keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie);
+int validate_config(const string& settingsFile, const map<string,Settings>& settings,
+ const string& configs);
+int validate_configs(const string& settingsFile, const map<string,Settings>& settings,
+ const vector<string>& configs);
+int select_files(vector<string> *resFiles, const string& config,
+ const map<string,Settings>& settings, const string& rootDir);
+int select_files(vector<vector<string> > *allResFiles, const vector<string>& configs,
+ const map<string,Settings>& settings, const string& rootDir);
+
+
+#endif // LOCALIZE_H
diff --git a/tools/localize/localize_test.cpp b/tools/localize/localize_test.cpp
new file mode 100644
index 0000000..63d904c
--- /dev/null
+++ b/tools/localize/localize_test.cpp
@@ -0,0 +1,219 @@
+#include "XLIFFFile.h"
+#include "ValuesFile.h"
+#include "localize.h"
+
+int pseudolocalize_xliff(XLIFFFile* xliff, bool expand);
+
+static int
+test_filename(const string& file, const string& locale, const string& expected)
+{
+ string result = translated_file_name(file, locale);
+ if (result != expected) {
+ fprintf(stderr, "translated_file_name test failed\n");
+ fprintf(stderr, " locale='%s'\n", locale.c_str());
+ fprintf(stderr, " expected='%s'\n", expected.c_str());
+ fprintf(stderr, " result='%s'\n", result.c_str());
+ return 1;
+ } else {
+ if (false) {
+ fprintf(stderr, "translated_file_name test passed\n");
+ fprintf(stderr, " locale='%s'\n", locale.c_str());
+ fprintf(stderr, " expected='%s'\n", expected.c_str());
+ fprintf(stderr, " result='%s'\n", result.c_str());
+ }
+ return 0;
+ }
+}
+
+static int
+translated_file_name_test()
+{
+ bool all = true;
+ int err = 0;
+
+ if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz_ZZ",
+ "//device/samples/NotePad/res/values-zz-rZZ/strings.xml");
+
+ if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz",
+ "//device/samples/NotePad/res/values-zz/strings.xml");
+
+ if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "",
+ "//device/samples/NotePad/res/values/strings.xml");
+
+ return err;
+}
+
+bool
+return_false(const string&, const TransUnit& unit, void* cookie)
+{
+ return false;
+}
+
+static int
+delete_trans_units()
+{
+ XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff");
+ if (xliff == NULL) {
+ printf("couldn't read file\n");
+ return 1;
+ }
+ if (false) {
+ printf("XLIFF was [[%s]]\n", xliff->ToString().c_str());
+ }
+
+ xliff->Filter(return_false, NULL);
+
+ if (false) {
+ printf("XLIFF is [[%s]]\n", xliff->ToString().c_str());
+
+ set<StringResource> const& strings = xliff->GetStringResources();
+ printf("strings.size=%zd\n", strings.size());
+ for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
+ const StringResource& str = *it;
+ printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(),
+ str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
+ str.pos.ToString().c_str(), str.file.c_str(), str.version,
+ str.versionString.c_str());
+ }
+ }
+
+ return 0;
+}
+
+static int
+filter_trans_units()
+{
+ XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff");
+ if (xliff == NULL) {
+ printf("couldn't read file\n");
+ return 1;
+ }
+
+ if (false) {
+ printf("XLIFF was [[%s]]\n", xliff->ToString().c_str());
+ }
+
+ Settings setting;
+ xliff->Filter(keep_this_trans_unit, &setting);
+
+ if (false) {
+ printf("XLIFF is [[%s]]\n", xliff->ToString().c_str());
+
+ set<StringResource> const& strings = xliff->GetStringResources();
+ printf("strings.size=%zd\n", strings.size());
+ for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
+ const StringResource& str = *it;
+ printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(),
+ str.value->ContentsToString(ANDROID_NAMESPACES).c_str(),
+ str.pos.ToString().c_str(), str.file.c_str(), str.version,
+ str.versionString.c_str());
+ }
+ }
+
+ return 0;
+}
+
+static int
+settings_test()
+{
+ int err;
+ map<string,Settings> settings;
+ map<string,Settings>::iterator it;
+
+ err = read_settings("testdata/config.xml", &settings, "//asdf");
+ if (err != 0) {
+ return err;
+ }
+
+ if (false) {
+ for (it=settings.begin(); it!=settings.end(); it++) {
+ const Settings& setting = it->second;
+ printf("CONFIG:\n");
+ printf(" id='%s'\n", setting.id.c_str());
+ printf(" oldVersion='%s'\n", setting.oldVersion.c_str());
+ printf(" currentVersion='%s'\n", setting.currentVersion.c_str());
+ int i=0;
+ for (vector<string>::const_iterator app=setting.apps.begin();
+ app!=setting.apps.end(); app++) {
+ printf(" apps[%02d]='%s'\n", i, app->c_str());
+ i++;
+ }
+ i=0;
+ for (vector<Reject>::const_iterator reject=setting.reject.begin();
+ reject!=setting.reject.end(); reject++) {
+ i++;
+ printf(" reject[%02d]=('%s','%s','%s')\n", i, reject->file.c_str(),
+ reject->name.c_str(), reject->comment.c_str());
+ }
+ }
+ }
+
+ for (it=settings.begin(); it!=settings.end(); it++) {
+ const Settings& setting = it->second;
+ if (it->first != setting.id) {
+ fprintf(stderr, "it->first='%s' setting.id='%s'\n", it->first.c_str(),
+ setting.id.c_str());
+ err |= 1;
+ }
+ }
+
+
+ return err;
+}
+
+static int
+test_one_pseudo(bool big, const char* expected)
+{
+ XLIFFFile* xliff = XLIFFFile::Parse("testdata/pseudo.xliff");
+ if (xliff == NULL) {
+ printf("couldn't read file\n");
+ return 1;
+ }
+ if (false) {
+ printf("XLIFF was [[%s]]\n", xliff->ToString().c_str());
+ }
+
+ pseudolocalize_xliff(xliff, big);
+ string newString = xliff->ToString();
+ delete xliff;
+
+ if (false) {
+ printf("XLIFF is [[%s]]\n", newString.c_str());
+ }
+
+ if (false && newString != expected) {
+ fprintf(stderr, "xliff didn't translate as expected\n");
+ fprintf(stderr, "newString=[[%s]]\n", newString.c_str());
+ fprintf(stderr, "expected=[[%s]]\n", expected);
+ return 1;
+ }
+
+ return 0;
+}
+
+static int
+pseudolocalize_test()
+{
+ int err = 0;
+
+ err |= test_one_pseudo(false, "");
+ //err |= test_one_pseudo(true, "");
+
+ return err;
+}
+
+int
+localize_test()
+{
+ bool all = true;
+ int err = 0;
+
+ if (all) err |= translated_file_name_test();
+ if (all) err |= delete_trans_units();
+ if (all) err |= filter_trans_units();
+ if (all) err |= settings_test();
+ if (all) err |= pseudolocalize_test();
+
+ return err;
+}
+
diff --git a/tools/localize/log.h b/tools/localize/log.h
new file mode 100644
index 0000000..4a5fa7f
--- /dev/null
+++ b/tools/localize/log.h
@@ -0,0 +1,7 @@
+#ifndef LOG_H
+#define LOG_H
+
+void log_printf(const char* fmt, ...);
+
+#endif // LOG_H
+
diff --git a/tools/localize/merge_res_and_xliff.cpp b/tools/localize/merge_res_and_xliff.cpp
new file mode 100644
index 0000000..58a6554
--- /dev/null
+++ b/tools/localize/merge_res_and_xliff.cpp
@@ -0,0 +1,391 @@
+#include "merge_res_and_xliff.h"
+
+#include "file_utils.h"
+#include "Perforce.h"
+#include "log.h"
+
+static set<StringResource>::const_iterator
+find_id(const set<StringResource>& s, const string& id, int index)
+{
+ for (set<StringResource>::const_iterator it = s.begin(); it != s.end(); it++) {
+ if (it->id == id && it->index == index) {
+ return it;
+ }
+ }
+ return s.end();
+}
+
+static set<StringResource>::const_iterator
+find_in_xliff(const set<StringResource>& s, const string& filename, const string& id, int index,
+ int version, const Configuration& config)
+{
+ for (set<StringResource>::const_iterator it = s.begin(); it != s.end(); it++) {
+ if (it->file == filename && it->id == id && it->index == index && it->version == version
+ && it->config == config) {
+ return it;
+ }
+ }
+ return s.end();
+}
+
+
+static void
+printit(const set<StringResource>& s, const set<StringResource>::const_iterator& it)
+{
+ if (it == s.end()) {
+ printf("(none)\n");
+ } else {
+ printf("id=%s index=%d config=%s file=%s value='%s'\n", it->id.c_str(), it->index,
+ it->config.ToString().c_str(), it->file.c_str(),
+ it->value->ToString(ANDROID_NAMESPACES).c_str());
+ }
+}
+
+StringResource
+convert_resource(const StringResource& s, const string& file, const Configuration& config,
+ int version, const string& versionString)
+{
+ return StringResource(s.pos, file, config, s.id, s.index, s.value ? s.value->Clone() : NULL,
+ version, versionString, s.comment);
+}
+
+static bool
+resource_has_contents(const StringResource& res)
+{
+ XMLNode* value = res.value;
+ if (value == NULL) {
+ return false;
+ }
+ string contents = value->ContentsToString(ANDROID_NAMESPACES);
+ return contents != "";
+}
+
+ValuesFile*
+merge_res_and_xliff(const ValuesFile* en_currentFile,
+ const ValuesFile* xx_currentFile, const ValuesFile* xx_oldFile,
+ const string& filename, const XLIFFFile* xliffFile)
+{
+ bool success = true;
+
+ Configuration en_config = xliffFile->SourceConfig();
+ Configuration xx_config = xliffFile->TargetConfig();
+ string currentVersion = xliffFile->CurrentVersion();
+
+ ValuesFile* result = new ValuesFile(xx_config);
+
+ set<StringResource> en_cur = en_currentFile->GetStrings();
+ set<StringResource> xx_cur = xx_currentFile->GetStrings();
+ set<StringResource> xx_old = xx_oldFile->GetStrings();
+ set<StringResource> xliff = xliffFile->GetStringResources();
+
+ // for each string in en_current
+ for (set<StringResource>::const_iterator en_c = en_cur.begin();
+ en_c != en_cur.end(); en_c++) {
+ set<StringResource>::const_iterator xx_c = find_id(xx_cur, en_c->id, en_c->index);
+ set<StringResource>::const_iterator xx_o = find_id(xx_old, en_c->id, en_c->index);
+ set<StringResource>::const_iterator xlf = find_in_xliff(xliff, en_c->file, en_c->id,
+ en_c->index, CURRENT_VERSION, xx_config);
+
+ if (false) {
+ printf("\nen_c: "); printit(en_cur, en_c);
+ printf("xx_c: "); printit(xx_cur, xx_c);
+ printf("xx_o: "); printit(xx_old, xx_o);
+ printf("xlf: "); printit(xliff, xlf);
+ }
+
+ // if it changed between xx_old and xx_current, use xx_current
+ // (someone changed it by hand)
+ if (xx_o != xx_old.end() && xx_c != xx_cur.end()) {
+ string xx_o_value = xx_o->value->ToString(ANDROID_NAMESPACES);
+ string xx_c_value = xx_c->value->ToString(ANDROID_NAMESPACES);
+ if (xx_o_value != xx_c_value && xx_c_value != "") {
+ StringResource r(convert_resource(*xx_c, filename, xx_config,
+ CURRENT_VERSION, currentVersion));
+ if (resource_has_contents(r)) {
+ result->AddString(r);
+ }
+ continue;
+ }
+ }
+
+ // if it is present in xliff, use that
+ // (it just got translated)
+ if (xlf != xliff.end() && xlf->value->ToString(ANDROID_NAMESPACES) != "") {
+ StringResource r(convert_resource(*xlf, filename, xx_config,
+ CURRENT_VERSION, currentVersion));
+ if (resource_has_contents(r)) {
+ result->AddString(r);
+ }
+ }
+
+ // if it is present in xx_current, use that
+ // (it was already translated, and not retranslated)
+ // don't filter out empty strings if they were added by hand, the above code just
+ // guarantees that this tool never adds an empty one.
+ if (xx_c != xx_cur.end()) {
+ StringResource r(convert_resource(*xx_c, filename, xx_config,
+ CURRENT_VERSION, currentVersion));
+ result->AddString(r);
+ }
+
+ // othwerwise, leave it out. The resource fall-through code will use the English
+ // one at runtime, and the xliff export code will pick it up for translation next time.
+ }
+
+ if (success) {
+ return result;
+ } else {
+ delete result;
+ return NULL;
+ }
+}
+
+
+struct MergedFile {
+ XLIFFFile* xliff;
+ string xliffFilename;
+ string original;
+ string translated;
+ ValuesFile* en_current;
+ ValuesFile* xx_current;
+ ValuesFile* xx_old;
+ ValuesFile* xx_new;
+ string xx_new_text;
+ string xx_new_filename;
+ bool new_file;
+ bool deleted_file;
+
+ MergedFile();
+ MergedFile(const MergedFile&);
+};
+
+struct compare_filenames {
+ bool operator()(const MergedFile& lhs, const MergedFile& rhs) const
+ {
+ return lhs.original < rhs.original;
+ }
+};
+
+MergedFile::MergedFile()
+ :xliff(NULL),
+ xliffFilename(),
+ original(),
+ translated(),
+ en_current(NULL),
+ xx_current(NULL),
+ xx_old(NULL),
+ xx_new(NULL),
+ xx_new_text(),
+ xx_new_filename(),
+ new_file(false),
+ deleted_file(false)
+{
+}
+
+MergedFile::MergedFile(const MergedFile& that)
+ :xliff(that.xliff),
+ xliffFilename(that.xliffFilename),
+ original(that.original),
+ translated(that.translated),
+ en_current(that.en_current),
+ xx_current(that.xx_current),
+ xx_old(that.xx_old),
+ xx_new(that.xx_new),
+ xx_new_text(that.xx_new_text),
+ xx_new_filename(that.xx_new_filename),
+ new_file(that.new_file),
+ deleted_file(that.deleted_file)
+{
+}
+
+
+typedef set<MergedFile, compare_filenames> MergedFileSet;
+
+int
+do_merge(const vector<string>& xliffFilenames)
+{
+ int err = 0;
+ MergedFileSet files;
+
+ printf("\rPreparing..."); fflush(stdout);
+ string currentChange = Perforce::GetCurrentChange(true);
+
+ // for each xliff, make a MergedFile record and do a little error checking
+ for (vector<string>::const_iterator xliffFilename=xliffFilenames.begin();
+ xliffFilename!=xliffFilenames.end(); xliffFilename++) {
+ XLIFFFile* xliff = XLIFFFile::Parse(*xliffFilename);
+ if (xliff == NULL) {
+ fprintf(stderr, "localize import: unable to read file %s\n", xliffFilename->c_str());
+ err = 1;
+ continue;
+ }
+
+ set<string> xf = xliff->Files();
+ for (set<string>::const_iterator f=xf.begin(); f!=xf.end(); f++) {
+ MergedFile mf;
+ mf.xliff = xliff;
+ mf.xliffFilename = *xliffFilename;
+ mf.original = *f;
+ mf.translated = translated_file_name(mf.original, xliff->TargetConfig().locale);
+ log_printf("mf.translated=%s mf.original=%s locale=%s\n", mf.translated.c_str(),
+ mf.original.c_str(), xliff->TargetConfig().locale.c_str());
+
+ if (files.find(mf) != files.end()) {
+ fprintf(stderr, "%s: duplicate string resources for file %s\n",
+ xliffFilename->c_str(), f->c_str());
+ fprintf(stderr, "%s: previously defined here.\n",
+ files.find(mf)->xliffFilename.c_str());
+ err = 1;
+ continue;
+ }
+ files.insert(mf);
+ }
+ }
+
+ size_t deletedFileCount = 0;
+ size_t J = files.size() * 3;
+ size_t j = 1;
+ // Read all of the files from perforce.
+ for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
+ MergedFile* file = const_cast<MergedFile*>(&(*mf));
+ // file->en_current
+ print_file_status(j++, J);
+ file->en_current = get_values_file(file->original, file->xliff->SourceConfig(),
+ CURRENT_VERSION, currentChange, true);
+ if (file->en_current == NULL) {
+ // deleted file
+ file->deleted_file = true;
+ deletedFileCount++;
+ continue;
+ }
+
+ // file->xx_current;
+ print_file_status(j++, J);
+ file->xx_current = get_values_file(file->translated, file->xliff->TargetConfig(),
+ CURRENT_VERSION, currentChange, false);
+ if (file->xx_current == NULL) {
+ file->xx_current = new ValuesFile(file->xliff->TargetConfig());
+ file->new_file = true;
+ }
+
+ // file->xx_old (note that the xliff's current version is our old version, because that
+ // was the current version when it was exported)
+ print_file_status(j++, J);
+ file->xx_old = get_values_file(file->translated, file->xliff->TargetConfig(),
+ OLD_VERSION, file->xliff->CurrentVersion(), false);
+ if (file->xx_old == NULL) {
+ file->xx_old = new ValuesFile(file->xliff->TargetConfig());
+ file->new_file = true;
+ }
+ }
+
+ // merge them
+ for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
+ MergedFile* file = const_cast<MergedFile*>(&(*mf));
+ if (file->deleted_file) {
+ continue;
+ }
+ file->xx_new = merge_res_and_xliff(file->en_current, file->xx_current, file->xx_old,
+ file->original, file->xliff);
+ }
+
+ // now is a good time to stop if there was an error
+ if (err != 0) {
+ return err;
+ }
+
+ // locate the files
+ j = 1;
+ for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
+ MergedFile* file = const_cast<MergedFile*>(&(*mf));
+ print_file_status(j++, J, "Locating");
+
+ file->xx_new_filename = Perforce::Where(file->translated, true);
+ if (file->xx_new_filename == "") {
+ fprintf(stderr, "\nWas not able to determine the location of depot file %s\n",
+ file->translated.c_str());
+ err = 1;
+ }
+ }
+
+ if (err != 0) {
+ return err;
+ }
+
+ // p4 edit the files
+ // only do this if it changed - no need to submit files that haven't changed meaningfully
+ vector<string> filesToEdit;
+ vector<string> filesToAdd;
+ vector<string> filesToDelete;
+ for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
+ MergedFile* file = const_cast<MergedFile*>(&(*mf));
+ if (file->deleted_file) {
+ filesToDelete.push_back(file->xx_new_filename);
+ continue;
+ }
+ string xx_current_text = file->xx_current->ToString();
+ string xx_new_text = file->xx_new->ToString();
+ if (xx_current_text != xx_new_text) {
+ if (file->xx_new->GetStrings().size() == 0) {
+ file->deleted_file = true;
+ filesToDelete.push_back(file->xx_new_filename);
+ } else {
+ file->xx_new_text = xx_new_text;
+ if (file->new_file) {
+ filesToAdd.push_back(file->xx_new_filename);
+ } else {
+ filesToEdit.push_back(file->xx_new_filename);
+ }
+ }
+ }
+ }
+ if (filesToAdd.size() == 0 && filesToEdit.size() == 0 && deletedFileCount == 0) {
+ printf("\nAll of the files are the same. Nothing to change.\n");
+ return 0;
+ }
+ if (filesToEdit.size() > 0) {
+ printf("\np4 editing files...\n");
+ if (0 != Perforce::EditFiles(filesToEdit, true)) {
+ return 1;
+ }
+ }
+
+
+ printf("\n");
+
+ for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) {
+ MergedFile* file = const_cast<MergedFile*>(&(*mf));
+ if (file->deleted_file) {
+ continue;
+ }
+ if (file->xx_new_text != "" && file->xx_new_filename != "") {
+ if (0 != write_to_file(file->xx_new_filename, file->xx_new_text)) {
+ err = 1;
+ }
+ }
+ }
+
+ if (err != 0) {
+ return err;
+ }
+
+ if (filesToAdd.size() > 0) {
+ printf("p4 adding %zd new files...\n", filesToAdd.size());
+ err = Perforce::AddFiles(filesToAdd, true);
+ }
+
+ if (filesToDelete.size() > 0) {
+ printf("p4 deleting %zd removed files...\n", filesToDelete.size());
+ err = Perforce::DeleteFiles(filesToDelete, true);
+ }
+
+ if (err != 0) {
+ return err;
+ }
+
+ printf("\n"
+ "Theoretically, this merge was successfull. Next you should\n"
+ "review the diffs, get a code review, and submit it. Enjoy.\n\n");
+ return 0;
+}
+
diff --git a/tools/localize/merge_res_and_xliff.h b/tools/localize/merge_res_and_xliff.h
new file mode 100644
index 0000000..acf2fff
--- /dev/null
+++ b/tools/localize/merge_res_and_xliff.h
@@ -0,0 +1,13 @@
+#ifndef MERGE_RES_AND_XLIFF_H
+#define MERGE_RES_AND_XLIFF_H
+
+#include "ValuesFile.h"
+#include "XLIFFFile.h"
+
+ValuesFile* merge_res_and_xliff(const ValuesFile* en_current,
+ const ValuesFile* xx_current, const ValuesFile* xx_old,
+ const string& filename, const XLIFFFile* xliff);
+
+int do_merge(const vector<string>& xliffFilenames);
+
+#endif // MERGE_RES_AND_XLIFF_H
diff --git a/tools/localize/merge_res_and_xliff_test.cpp b/tools/localize/merge_res_and_xliff_test.cpp
new file mode 100644
index 0000000..5a2b0f4
--- /dev/null
+++ b/tools/localize/merge_res_and_xliff_test.cpp
@@ -0,0 +1,47 @@
+#include "merge_res_and_xliff.h"
+
+
+int
+merge_test()
+{
+ Configuration english;
+ english.locale = "en_US";
+ Configuration translated;
+ translated.locale = "zz_ZZ";
+
+ ValuesFile* en_current = ValuesFile::ParseFile("testdata/merge_en_current.xml", english,
+ CURRENT_VERSION, "3");
+ if (en_current == NULL) {
+ fprintf(stderr, "merge_test: unable to read testdata/merge_en_current.xml\n");
+ return 1;
+ }
+
+ ValuesFile* xx_current = ValuesFile::ParseFile("testdata/merge_xx_current.xml", translated,
+ CURRENT_VERSION, "3");
+ if (xx_current == NULL) {
+ fprintf(stderr, "merge_test: unable to read testdata/merge_xx_current.xml\n");
+ return 1;
+ }
+ ValuesFile* xx_old = ValuesFile::ParseFile("testdata/merge_xx_old.xml", translated,
+ OLD_VERSION, "2");
+ if (xx_old == NULL) {
+ fprintf(stderr, "merge_test: unable to read testdata/merge_xx_old.xml\n");
+ return 1;
+ }
+
+ XLIFFFile* xliff = XLIFFFile::Parse("testdata/merge.xliff");
+
+ ValuesFile* result = merge_res_and_xliff(en_current, xx_current, xx_old,
+ "//device/tools/localize/testdata/res/values/strings.xml", xliff);
+
+ if (result == NULL) {
+ fprintf(stderr, "merge_test: result is NULL\n");
+ return 1;
+ }
+
+ printf("======= RESULT =======\n%s===============\n", result->ToString().c_str());
+
+ return 0;
+}
+
+
diff --git a/tools/localize/res_check.cpp b/tools/localize/res_check.cpp
new file mode 100644
index 0000000..0fab98a
--- /dev/null
+++ b/tools/localize/res_check.cpp
@@ -0,0 +1,106 @@
+#include "res_check.h"
+#include "localize.h"
+#include "file_utils.h"
+#include "ValuesFile.h"
+
+#include <stdio.h>
+
+static int check_file(const ValuesFile* file);
+static int check_value(const SourcePos& pos, const XMLNode* value);
+static int scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth = 0);
+
+int
+do_rescheck(const vector<string>& files)
+{
+ int err;
+
+ Configuration english;
+ english.locale = "en_US";
+
+ for (size_t i=0; i<files.size(); i++) {
+ const string filename = files[i];
+ ValuesFile* valuesFile = get_local_values_file(filename, english, CURRENT_VERSION,
+ "0", true);
+ if (valuesFile != NULL) {
+ err |= check_file(valuesFile);
+ delete valuesFile;
+ } else {
+ err |= 1;
+ }
+ }
+
+ return err;
+}
+
+static int
+check_file(const ValuesFile* file)
+{
+ int err = 0;
+ set<StringResource> strings = file->GetStrings();
+ for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) {
+ XMLNode* value = it->value;
+ if (value != NULL) {
+ err |= check_value(it->pos, value);
+ }
+ }
+ return err;
+}
+
+static bool
+contains_percent(const string& str)
+{
+ const size_t len = str.length();
+ for (size_t i=0; i<len; i++) {
+ char c = str[i];
+ if (c == '%') {
+ return true;
+ }
+ }
+ return false;
+}
+
+static int
+check_value(const SourcePos& pos, const XMLNode* value)
+{
+ int err = 0;
+ err |= scan_for_unguarded_format(pos, value);
+ return err;
+}
+
+static bool
+is_xliff_block(const string& ns, const string& name)
+{
+ if (ns == XLIFF_XMLNS) {
+ return name == "g";
+ } else {
+ return false;
+ }
+}
+
+static int
+scan_for_unguarded_format(const SourcePos& pos, const string& string)
+{
+ bool containsPercent = contains_percent(string);
+ if (containsPercent) {
+ pos.Error("unguarded percent: '%s'\n", string.c_str());
+ }
+ return 0;
+}
+
+static int
+scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth)
+{
+ if (value->Type() == XMLNode::ELEMENT) {
+ int err = 0;
+ if (depth == 0 || !is_xliff_block(value->Namespace(), value->Name())) {
+ const vector<XMLNode*>& children = value->Children();
+ for (size_t i=0; i<children.size(); i++) {
+ err |= scan_for_unguarded_format(pos, children[i], depth+1);
+ }
+ }
+ return err;
+ } else {
+ return scan_for_unguarded_format(pos, value->Text());
+ }
+}
+
diff --git a/tools/localize/res_check.h b/tools/localize/res_check.h
new file mode 100644
index 0000000..86e7ce6
--- /dev/null
+++ b/tools/localize/res_check.h
@@ -0,0 +1,12 @@
+#ifndef RESCHECK_H
+#define RESCHECK_H
+
+#include <map>
+#include <string>
+#include <vector>
+
+using namespace std;
+
+int do_rescheck(const vector<string>& files);
+
+#endif // RESCHECK_H
diff --git a/tools/localize/test.cpp b/tools/localize/test.cpp
new file mode 100644
index 0000000..5fa2c17
--- /dev/null
+++ b/tools/localize/test.cpp
@@ -0,0 +1,31 @@
+#include "SourcePos.h"
+#include <stdio.h>
+
+int ValuesFile_test();
+int XLIFFFile_test();
+int XMLHandler_test();
+int Perforce_test();
+int localize_test();
+int merge_test();
+
+int
+test()
+{
+ bool all = true;
+ int err = 0;
+
+ if (all) err |= XMLHandler_test();
+ if (all) err |= ValuesFile_test();
+ if (all) err |= XLIFFFile_test();
+ if (all) err |= Perforce_test();
+ if (all) err |= localize_test();
+ if (all) err |= merge_test();
+
+ if (err != 0) {
+ fprintf(stderr, "some tests failed\n");
+ } else {
+ fprintf(stderr, "all tests passed\n");
+ }
+
+ return err;
+}
diff --git a/tools/localize/testdata/config.xml b/tools/localize/testdata/config.xml
new file mode 100644
index 0000000..affa140
--- /dev/null
+++ b/tools/localize/testdata/config.xml
@@ -0,0 +1,15 @@
+<localize-config>
+ <configuration id="system"
+ old-cl="1"
+ new-cl="43019">
+ <app dir="apps/common" />
+ </configuration>
+ <configuration id="samples"
+ old-cl="24801"
+ new-cl="43019">
+ <app dir="samples/NotePad" />
+ <reject file="samples/NotePad/res/values/strings.xml" name="string:menu_delete">
+ QA says this sounds <b>rude</b>.
+ </reject>
+ </configuration>
+</localize-config>
diff --git a/tools/localize/testdata/import.xliff b/tools/localize/testdata/import.xliff
new file mode 100644
index 0000000..b99b739
--- /dev/null
+++ b/tools/localize/testdata/import.xliff
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
+ version="1.2"
+ >
+ <file datatype="x-android-res"
+ original="//device/tools/localize/testdata/res/values/strings.xml"
+ product-version="1.0"
+ date="08:10:54 12/07/07 PST"
+ source-language="en_US"
+ product-name="kila"
+ target-language="zz_ZZ"
+ build-num="44391"
+ >
+ <body>
+ <trans-unit id="string:changed_in_xx">
+ <source>aaa</source>
+ <target>AAA</target>
+ </trans-unit>
+ <trans-unit id="string:first_translation">
+ <source>bbb</source>
+ <target>BBB</target>
+ </trans-unit>
+ <trans-unit id="string:deleted_string">
+ <source>ddd</source>
+ <target>DDDD</target>
+ </trans-unit>
+ <trans-unit id="array:0:growing_array">
+ <source>1-One</source>
+ <target>1-oNE</target>
+ </trans-unit>
+ <trans-unit id="array:1:growing_array">
+ <source>1-Two</source>
+ <target>1-tWO</target>
+ </trans-unit>
+ <trans-unit id="array:2:growing_array">
+ <source>1-Three</source>
+ <target>1-tHREE</target>
+ </trans-unit>
+ <trans-unit id="array:0:shrinking_array">
+ <source>2-One</source>
+ <target>2-oNE</target>
+ </trans-unit>
+ <trans-unit id="array:1:shrinking_array">
+ <source>2-Two</source>
+ <target>2-tWO</target>
+ </trans-unit>
+ <trans-unit id="array:2:shrinking_array">
+ <source>2-Three</source>
+ <target>2-tHREE</target>
+ </trans-unit>
+ <trans-unit id="array:3:shrinking_array">
+ <source>2-Four</source>
+ <target>2-fOUR</target>
+ </trans-unit>
+ <trans-unit id="array:0:deleted_array">
+ <source>4-One</source>
+ <target>4-oNE</target>
+ </trans-unit>
+ <trans-unit id="array:1:deleted_array">
+ <source>4-Two</source>
+ <target>4-tWO</target>
+ </trans-unit>
+ <trans-unit id="array:2:deleted_array">
+ <source>4-Three</source>
+ <target>4-tHREE</target>
+ </trans-unit>
+
+ </body>
+ </file>
+</xliff>
+
+
diff --git a/tools/localize/testdata/merge.xliff b/tools/localize/testdata/merge.xliff
new file mode 100644
index 0000000..2b78c45
--- /dev/null
+++ b/tools/localize/testdata/merge.xliff
@@ -0,0 +1,72 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
+ version="1.2"
+ >
+ <file datatype="x-android-res"
+ original="testdata/merge_en_current.xml"
+ product-version="1.0"
+ date="08:10:54 12/07/07 PST"
+ source-language="en-US"
+ product-name="kila"
+ target-language="zz-ZZ"
+ build-num="44391"
+ >
+ <body>
+ <trans-unit id="string:changed_in_xx">
+ <source>aaa</source>
+ <target>AAA</target>
+ </trans-unit>
+ <trans-unit id="string:first_translation">
+ <source>bbb</source>
+ <target>BBB</target>
+ </trans-unit>
+ <trans-unit id="string:deleted_string">
+ <source>ddd</source>
+ <target>DDDD</target>
+ </trans-unit>
+ <trans-unit id="array:0:growing_array">
+ <source>1-One</source>
+ <target>1-oNE</target>
+ </trans-unit>
+ <trans-unit id="array:1:growing_array">
+ <source>1-Two</source>
+ <target>1-tWO</target>
+ </trans-unit>
+ <trans-unit id="array:2:growing_array">
+ <source>1-Three</source>
+ <target>1-tHREE</target>
+ </trans-unit>
+ <trans-unit id="array:0:shrinking_array">
+ <source>2-One</source>
+ <target>2-oNE</target>
+ </trans-unit>
+ <trans-unit id="array:1:shrinking_array">
+ <source>2-Two</source>
+ <target>2-tWO</target>
+ </trans-unit>
+ <trans-unit id="array:2:shrinking_array">
+ <source>2-Three</source>
+ <target>2-tHREE</target>
+ </trans-unit>
+ <trans-unit id="array:3:shrinking_array">
+ <source>2-Four</source>
+ <target>2-fOUR</target>
+ </trans-unit>
+ <trans-unit id="array:0:deleted_array">
+ <source>4-One</source>
+ <target>4-oNE</target>
+ </trans-unit>
+ <trans-unit id="array:1:deleted_array">
+ <source>4-Two</source>
+ <target>4-tWO</target>
+ </trans-unit>
+ <trans-unit id="array:2:deleted_array">
+ <source>4-Three</source>
+ <target>4-tHREE</target>
+ </trans-unit>
+
+ </body>
+ </file>
+</xliff>
+
+
diff --git a/tools/localize/testdata/merge_en_current.xml b/tools/localize/testdata/merge_en_current.xml
new file mode 100644
index 0000000..6a11e68
--- /dev/null
+++ b/tools/localize/testdata/merge_en_current.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="changed_in_xx">aaa</string>
+ <string name="first_translation">bbb</string>
+ <string name="previously_translated">ccc</string>
+ <string name="new_string">ccc</string>
+
+ <string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string>
+
+ <array name="growing_array">
+ <!-- somebody wrote a comment! -->
+ <item>1-One</item>
+ <item>1-Two</item>
+ <item>1-Three</item>
+ <item>1-Four</item>
+ </array>
+ <array name="shrinking_array">
+ <!-- somebody wrote a comment! -->
+ <item>2-One</item>
+ <item>2-Two</item>
+ <item>2-Three</item>
+ </array>
+ <array name="new_array">
+ <!-- somebody wrote a comment! -->
+ <item>3-One</item>
+ <item>3-Two</item>
+ <item>3-Three</item>
+ </array>
+</resources>
diff --git a/tools/localize/testdata/merge_en_old.xml b/tools/localize/testdata/merge_en_old.xml
new file mode 100644
index 0000000..933f98e
--- /dev/null
+++ b/tools/localize/testdata/merge_en_old.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="changed_in_xx">aaa</string>
+ <string name="first_translation">bbb</string>
+ <string name="previously_translated">ccc</string>
+ <string name="deleted_string">ddd</string>
+
+ <string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string>
+
+ <array name="growing_array">
+ <!-- somebody wrote a comment! -->
+ <item>1-One</item>
+ <item>1-Two</item>
+ <item>1-Three</item>
+ </array>
+ <array name="shrinking_array">
+ <!-- somebody wrote a comment! -->
+ <item>2-One</item>
+ <item>2-Two</item>
+ <item>2-Three</item>
+ <item>2-Four</item>
+ </array>
+ <array name="deleted_array">
+ <!-- somebody wrote a comment! -->
+ <item>4-One</item>
+ <item>4-Two</item>
+ <item>4-Three</item>
+ </array>
+</resources>
+
diff --git a/tools/localize/testdata/merge_xx_current.xml b/tools/localize/testdata/merge_xx_current.xml
new file mode 100644
index 0000000..c2a783d
--- /dev/null
+++ b/tools/localize/testdata/merge_xx_current.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="changed_in_xx">AAAA</string>
+ <string name="previously_translated">CCC</string>
+</resources>
+
+
diff --git a/tools/localize/testdata/merge_xx_old.xml b/tools/localize/testdata/merge_xx_old.xml
new file mode 100644
index 0000000..9d3a7d8
--- /dev/null
+++ b/tools/localize/testdata/merge_xx_old.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="changed_in_xx">aaa</string>
+ <string name="previously_translated">CCC</string>
+</resources>
+
diff --git a/tools/localize/testdata/pseudo.xliff b/tools/localize/testdata/pseudo.xliff
new file mode 100644
index 0000000..5b44f86
--- /dev/null
+++ b/tools/localize/testdata/pseudo.xliff
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
+ version="1.2"
+ >
+ <file datatype="x-android-res"
+ original="//device/tools/localization/tests/res/values/strings.xml"
+ product-version="1.0"
+ date="08:10:54 12/07/07 PST"
+ source-language="en-US"
+ product-name="kila"
+ target-language="zz-ZZ"
+ build-num="32138"
+ >
+ <body>
+ <trans-unit id="string:complex">
+ <source>First <g id="string:complex:0" ctype="underline">underline</g>, <g id="string:complex:1" ctype="italic">italic<g id="string:complex:2" ctype="bold">italicbold</g></g> End </source>
+ </trans-unit>
+ <trans-unit id="string:complex-quoted">
+ <source xml:space="preserve">First <g id="string:complex-quoted:0" ctype="underline">underline</g>, <g id="string:complex-quoted:1" ctype="italic">italic<g id="string:complex-quoted:2" ctype="bold">italicbold</g></g> End</source>
+ </trans-unit>
+ <trans-unit id="string:simple">
+ <source>Simple</source>
+ </trans-unit>
+ <trans-unit id="array:0:simple">
+ <source>Simple</source>
+ </trans-unit>
+ <trans-unit id="array:1:simple">
+ <source>Simple</source>
+ </trans-unit>
+ <trans-unit id="string:simple-quoted">
+ <source xml:space="preserve"> Quote</source>
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve"> OLD Quote</source>
+ <target xml:lang="xx"> OLD Ờũỡŧę</target>
+ </alt-trans>
+ </trans-unit>
+ </body>
+ </file>
+</xliff>
+
diff --git a/tools/localize/testdata/res/values-zz-rZZ/strings.xml b/tools/localize/testdata/res/values-zz-rZZ/strings.xml
new file mode 100644
index 0000000..c2a783d
--- /dev/null
+++ b/tools/localize/testdata/res/values-zz-rZZ/strings.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="changed_in_xx">AAAA</string>
+ <string name="previously_translated">CCC</string>
+</resources>
+
+
diff --git a/tools/localize/testdata/res/values/strings.xml b/tools/localize/testdata/res/values/strings.xml
new file mode 100644
index 0000000..6a11e68
--- /dev/null
+++ b/tools/localize/testdata/res/values/strings.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="changed_in_xx">aaa</string>
+ <string name="first_translation">bbb</string>
+ <string name="previously_translated">ccc</string>
+ <string name="new_string">ccc</string>
+
+ <string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string>
+
+ <array name="growing_array">
+ <!-- somebody wrote a comment! -->
+ <item>1-One</item>
+ <item>1-Two</item>
+ <item>1-Three</item>
+ <item>1-Four</item>
+ </array>
+ <array name="shrinking_array">
+ <!-- somebody wrote a comment! -->
+ <item>2-One</item>
+ <item>2-Two</item>
+ <item>2-Three</item>
+ </array>
+ <array name="new_array">
+ <!-- somebody wrote a comment! -->
+ <item>3-One</item>
+ <item>3-Two</item>
+ <item>3-Three</item>
+ </array>
+</resources>
diff --git a/tools/localize/testdata/strip_xliff.xliff b/tools/localize/testdata/strip_xliff.xliff
new file mode 100644
index 0000000..9254cf2
--- /dev/null
+++ b/tools/localize/testdata/strip_xliff.xliff
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
+ version="1.2"
+ >
+ <file datatype="x-android-res"
+ original="//device/tools/localization/tests/res/values/strings.xml"
+ product-version="1.0"
+ date="08:10:54 12/07/07 PST"
+ source-language="en-US"
+ product-name="kila"
+ target-language="zz-ZZ"
+ build-num="32138"
+ >
+ <body>
+
+ <trans-unit id="string:string-000-0">
+ </trans-unit>
+ <trans-unit id="string:string-001-0">
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve">source</source>
+ </alt-trans>
+ </trans-unit>
+ <trans-unit id="string:string-010-0">
+ <alt-trans>
+ <target xml:lang="zz" xml:space="preserve">target</target>
+ </alt-trans>
+ </trans-unit>
+ <trans-unit id="string:string-011-0">
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve">source</source>
+ <target xml:lang="zz" xml:space="preserve">target</target>
+ </alt-trans>
+ </trans-unit>
+
+ <trans-unit id="string:string-100-1">
+ <source xml:space="preserve">source</source>
+ </trans-unit>
+ <trans-unit id="string:string-101-1">
+ <source xml:space="preserve">source</source>
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve">source</source>
+ </alt-trans>
+ </trans-unit>
+ <trans-unit id="string:string-110-1">
+ <source xml:space="preserve">source</source>
+ <alt-trans>
+ <target xml:lang="zz" xml:space="preserve">target</target>
+ </alt-trans>
+ </trans-unit>
+
+ <trans-unit id="string:string-111-0">
+ <source xml:space="preserve">source</source>
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve">source</source>
+ <target xml:lang="zz" xml:space="preserve">target</target>
+ </alt-trans>
+ </trans-unit>
+ <trans-unit id="string:string-111-1">
+ <source xml:space="preserve">source</source>
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve">alt-source</source>
+ <target xml:lang="zz" xml:space="preserve">target</target>
+ </alt-trans>
+ </trans-unit>
+
+ </body>
+ </file>
+</xliff>
+
+
diff --git a/tools/localize/testdata/values/strings.xml b/tools/localize/testdata/values/strings.xml
new file mode 100644
index 0000000..5e8d43d
--- /dev/null
+++ b/tools/localize/testdata/values/strings.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Copyright (C) 2007 The Android Open Source Project
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources
+ xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">
+ <string name="test1">Discard</string>
+ <!-- comment -->
+ <string name="test2">a<b>b<i>c</i></b>d</string>
+ <string name="test3">a<xliff:g a="b" xliff:a="asdf">bBb</xliff:g>C</string>
+
+ <!-- Email address types from android.provider.Contacts -->
+ <array name="emailAddressTypes">
+ <!-- somebody wrote a comment! -->
+ <item>Email</item>
+ <item>Home</item>
+ <item>Work</item>
+ <item>Other\u2026</item>
+ </array>
+</resources>
diff --git a/tools/localize/testdata/xliff1.xliff b/tools/localize/testdata/xliff1.xliff
new file mode 100644
index 0000000..55a8d8e
--- /dev/null
+++ b/tools/localize/testdata/xliff1.xliff
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2"
+ version="1.2"
+ >
+ <file datatype="x-android-res"
+ original="//device/tools/localization/tests/res/values/strings.xml"
+ product-version="1.0"
+ date="08:10:54 12/07/07 PST"
+ source-language="en-US"
+ product-name="kila"
+ target-language="zz-ZZ"
+ build-num="32138"
+ >
+ <body>
+ <trans-unit id="string:complex">
+ <source>First <g id="string:complex:0" ctype="underline">underline</g>, <g id="string:complex:1" ctype="italic">italic<g id="string:complex:2" ctype="bold">italicbold</g></g> End </source>
+ <target>Ḟịṙṩŧ , Ḛŋḋ </target>
+ </trans-unit>
+ <trans-unit id="string:complex-quoted">
+ <source xml:space="preserve">First <g id="string:complex-quoted:0" ctype="underline">underline</g>, <g id="string:complex-quoted:1" ctype="italic">italic<g id="string:complex-quoted:2" ctype="bold">italicbold</g></g> End</source>
+ <target>Ḟịṙṩŧ , Ḛŋḋ</target>
+ </trans-unit>
+ <trans-unit id="string:simple">
+ <source>Simple</source>
+ <target>Ṩịṃṕļę</target>
+ </trans-unit>
+ <trans-unit id="array:0:simple">
+ <source>Simple</source>
+ <target>Ṩịṃṕļę</target>
+ </trans-unit>
+ <trans-unit id="array:1:simple">
+ <source>Simple</source>
+ <target>Ṩịṃṕļę</target>
+ </trans-unit>
+ <trans-unit id="string:simple-quoted">
+ <source xml:space="preserve"> Quote</source>
+ <target> Ờũỡŧę</target>
+ <alt-trans>
+ <source xml:lang="en" xml:space="preserve"> OLD Quote</source>
+ <target xml:lang="xx"> OLD Ờũỡŧę</target>
+ </alt-trans>
+ </trans-unit>
+ </body>
+ </file>
+</xliff>
+
diff --git a/tools/localize/testdata/xml.xml b/tools/localize/testdata/xml.xml
new file mode 100644
index 0000000..ef930d0
--- /dev/null
+++ b/tools/localize/testdata/xml.xml
@@ -0,0 +1,16 @@
+<ASDF>
+ <a id="system"
+ old-cl="1"
+ new-cl="43019">
+ <app dir="apps/common" />
+ </a>
+ <a id="samples"
+ old-cl="1"
+ new-cl="43019">asdf
+ <app dir="samples/NotePad" />
+ <app dir="samples/LunarLander" />
+ <something>a<b>,</b>b </something>
+ <exact xml:space="preserve">a<b>,</b>b </exact>
+ </a>
+</ASDF>
+
diff --git a/tools/localize/xmb.cpp b/tools/localize/xmb.cpp
new file mode 100644
index 0000000..236705f
--- /dev/null
+++ b/tools/localize/xmb.cpp
@@ -0,0 +1,181 @@
+#include "xmb.h"
+
+#include "file_utils.h"
+#include "localize.h"
+#include "ValuesFile.h"
+#include "XMLHandler.h"
+#include "XLIFFFile.h"
+
+#include <map>
+
+using namespace std;
+
+const char *const NS_MAP[] = {
+ "xml", XMLNS_XMLNS,
+ NULL, NULL
+};
+
+set<string> g_tags;
+
+static string
+strip_newlines(const string& str)
+{
+ string res;
+ const size_t N = str.length();
+ for (size_t i=0; i<N; i++) {
+ char c = str[i];
+ if (c != '\n' && c != '\r') {
+ res += c;
+ } else {
+ res += ' ';
+ }
+ }
+ return res;
+}
+
+static int
+rename_id_attribute(XMLNode* node)
+{
+ vector<XMLAttribute>& attrs = node->EditAttributes();
+ const size_t I = attrs.size();
+ for (size_t i=0; i<I; i++) {
+ XMLAttribute attr = attrs[i];
+ if (attr.name == "id") {
+ attr.name = "name";
+ attrs.erase(attrs.begin()+i);
+ attrs.push_back(attr);
+ return 0;
+ }
+ }
+ return 1;
+}
+
+static int
+convert_xliff_to_ph(XMLNode* node, int* phID)
+{
+ int err = 0;
+ if (node->Type() == XMLNode::ELEMENT) {
+ if (node->Namespace() == XLIFF_XMLNS) {
+ g_tags.insert(node->Name());
+ node->SetName("", "ph");
+
+ err = rename_id_attribute(node);
+ if (err != 0) {
+ char name[30];
+ (*phID)++;
+ sprintf(name, "id-%d", *phID);
+ node->EditAttributes().push_back(XMLAttribute("", "name", name));
+ err = 0;
+ }
+ }
+ vector<XMLNode*>& children = node->EditChildren();
+ const size_t I = children.size();
+ for (size_t i=0; i<I; i++) {
+ err |= convert_xliff_to_ph(children[i], phID);
+ }
+ }
+ return err;
+}
+
+XMLNode*
+resource_to_xmb_msg(const StringResource& res)
+{
+ // the msg element
+ vector<XMLAttribute> attrs;
+ string name = res.pos.file;
+ name += ":";
+ name += res.TypedID();
+ attrs.push_back(XMLAttribute("", "name", name));
+ attrs.push_back(XMLAttribute("", "desc", strip_newlines(res.comment)));
+ attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve"));
+ XMLNode* msg = XMLNode::NewElement(res.pos, "", "msg", attrs, XMLNode::EXACT);
+
+ // the contents are in xliff/html, convert it to xliff
+ int err = 0;
+ XMLNode* value = res.value;
+ string tag = value->Name();
+ int phID = 0;
+ for (vector<XMLNode*>::const_iterator it=value->Children().begin();
+ it!=value->Children().end(); it++) {
+ err |= convert_html_to_xliff(*it, tag, msg, &phID);
+ }
+
+ if (err != 0) {
+ return NULL;
+ }
+
+ // and then convert that to xmb
+ for (vector<XMLNode*>::iterator it=msg->EditChildren().begin();
+ it!=msg->EditChildren().end(); it++) {
+ err |= convert_xliff_to_ph(*it, &phID);
+ }
+
+ if (err == 0) {
+ return msg;
+ } else {
+ return NULL;
+ }
+}
+
+int
+do_xlb_export(const string& outfile, const vector<string>& resFiles)
+{
+ int err = 0;
+
+ size_t totalFileCount = resFiles.size();
+
+ Configuration english;
+ english.locale = "en_US";
+
+ set<StringResource> allResources;
+
+ const size_t J = resFiles.size();
+ for (size_t j=0; j<J; j++) {
+ string resFile = resFiles[j];
+
+ ValuesFile* valuesFile = get_local_values_file(resFile, english, CURRENT_VERSION, "", true);
+ if (valuesFile != NULL) {
+ set<StringResource> resources = valuesFile->GetStrings();
+ allResources.insert(resources.begin(), resources.end());
+ } else {
+ fprintf(stderr, "error reading file %s\n", resFile.c_str());
+ }
+
+ delete valuesFile;
+ }
+
+ // Construct the XLB xml
+ vector<XMLAttribute> attrs;
+ attrs.push_back(XMLAttribute("", "locale", "en"));
+ XMLNode* localizationbundle = XMLNode::NewElement(GENERATED_POS, "", "localizationbundle",
+ attrs, XMLNode::PRETTY);
+
+ for (set<StringResource>::iterator it=allResources.begin(); it!=allResources.end(); it++) {
+ XMLNode* msg = resource_to_xmb_msg(*it);
+ if (msg) {
+ localizationbundle->EditChildren().push_back(msg);
+ } else {
+ err = 1;
+ }
+ }
+
+#if 0
+ for (set<string>::iterator it=g_tags.begin(); it!=g_tags.end(); it++) {
+ printf("tag: %s\n", it->c_str());
+ }
+ printf("err=%d\n", err);
+#endif
+ if (err == 0) {
+ FILE* f = fopen(outfile.c_str(), "wb");
+ if (f == NULL) {
+ fprintf(stderr, "can't open outputfile: %s\n", outfile.c_str());
+ return 1;
+ }
+ fprintf(f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
+ fprintf(f, "%s\n", localizationbundle->ToString(NS_MAP).c_str());
+ fclose(f);
+ }
+
+ return err;
+}
+
diff --git a/tools/localize/xmb.h b/tools/localize/xmb.h
new file mode 100644
index 0000000..96492b1
--- /dev/null
+++ b/tools/localize/xmb.h
@@ -0,0 +1,11 @@
+#ifndef XMB_H
+#define XMB_H
+
+#include <string>
+#include <vector>
+
+using namespace std;
+
+int do_xlb_export(const string& outFile, const vector<string>& resFiles);
+
+#endif // XMB_H