From 9066cfe9886ac131c34d59ed0e2d287b0e3c0087 Mon Sep 17 00:00:00 2001 From: The Android Open Source Project Date: Tue, 3 Mar 2009 19:31:44 -0800 Subject: auto import from //depot/cupcake/@135843 --- tools/localize/Android.mk | 56 ++ tools/localize/Configuration.cpp | 76 ++ tools/localize/Configuration.h | 38 + tools/localize/Perforce.cpp | 230 ++++++ tools/localize/Perforce.h | 25 + tools/localize/Perforce_test.cpp | 62 ++ tools/localize/SourcePos.cpp | 166 +++++ tools/localize/SourcePos.h | 28 + tools/localize/Values.cpp | 134 ++++ tools/localize/Values.h | 48 ++ tools/localize/ValuesFile.cpp | 266 +++++++ tools/localize/ValuesFile.h | 52 ++ tools/localize/ValuesFile_test.cpp | 54 ++ tools/localize/XLIFFFile.cpp | 609 ++++++++++++++++ tools/localize/XLIFFFile.h | 98 +++ tools/localize/XLIFFFile_test.cpp | 115 +++ tools/localize/XMLHandler.cpp | 793 +++++++++++++++++++++ tools/localize/XMLHandler.h | 197 +++++ tools/localize/XMLHandler_test.cpp | 133 ++++ tools/localize/XMLNode.h | 19 + tools/localize/file_utils.cpp | 143 ++++ tools/localize/file_utils.h | 21 + tools/localize/localize.cpp | 767 ++++++++++++++++++++ tools/localize/localize.h | 40 ++ tools/localize/localize_test.cpp | 219 ++++++ tools/localize/log.h | 7 + tools/localize/merge_res_and_xliff.cpp | 391 ++++++++++ tools/localize/merge_res_and_xliff.h | 13 + tools/localize/merge_res_and_xliff_test.cpp | 47 ++ tools/localize/res_check.cpp | 106 +++ tools/localize/res_check.h | 12 + tools/localize/test.cpp | 31 + tools/localize/testdata/config.xml | 15 + tools/localize/testdata/import.xliff | 72 ++ tools/localize/testdata/merge.xliff | 72 ++ tools/localize/testdata/merge_en_current.xml | 44 ++ tools/localize/testdata/merge_en_old.xml | 45 ++ tools/localize/testdata/merge_xx_current.xml | 22 + tools/localize/testdata/merge_xx_old.xml | 21 + tools/localize/testdata/pseudo.xliff | 40 ++ .../testdata/res/values-zz-rZZ/strings.xml | 22 + tools/localize/testdata/res/values/strings.xml | 44 ++ tools/localize/testdata/strip_xliff.xliff | 70 ++ tools/localize/testdata/values/strings.xml | 32 + tools/localize/testdata/xliff1.xliff | 46 ++ tools/localize/testdata/xml.xml | 16 + tools/localize/xmb.cpp | 181 +++++ tools/localize/xmb.h | 11 + 48 files changed, 5749 insertions(+) create mode 100644 tools/localize/Android.mk create mode 100644 tools/localize/Configuration.cpp create mode 100644 tools/localize/Configuration.h create mode 100644 tools/localize/Perforce.cpp create mode 100644 tools/localize/Perforce.h create mode 100644 tools/localize/Perforce_test.cpp create mode 100644 tools/localize/SourcePos.cpp create mode 100644 tools/localize/SourcePos.h create mode 100644 tools/localize/Values.cpp create mode 100644 tools/localize/Values.h create mode 100644 tools/localize/ValuesFile.cpp create mode 100644 tools/localize/ValuesFile.h create mode 100644 tools/localize/ValuesFile_test.cpp create mode 100644 tools/localize/XLIFFFile.cpp create mode 100644 tools/localize/XLIFFFile.h create mode 100644 tools/localize/XLIFFFile_test.cpp create mode 100644 tools/localize/XMLHandler.cpp create mode 100644 tools/localize/XMLHandler.h create mode 100644 tools/localize/XMLHandler_test.cpp create mode 100644 tools/localize/XMLNode.h create mode 100644 tools/localize/file_utils.cpp create mode 100644 tools/localize/file_utils.h create mode 100644 tools/localize/localize.cpp create mode 100644 tools/localize/localize.h create mode 100644 tools/localize/localize_test.cpp create mode 100644 tools/localize/log.h create mode 100644 tools/localize/merge_res_and_xliff.cpp create mode 100644 tools/localize/merge_res_and_xliff.h create mode 100644 tools/localize/merge_res_and_xliff_test.cpp create mode 100644 tools/localize/res_check.cpp create mode 100644 tools/localize/res_check.h create mode 100644 tools/localize/test.cpp create mode 100644 tools/localize/testdata/config.xml create mode 100644 tools/localize/testdata/import.xliff create mode 100644 tools/localize/testdata/merge.xliff create mode 100644 tools/localize/testdata/merge_en_current.xml create mode 100644 tools/localize/testdata/merge_en_old.xml create mode 100644 tools/localize/testdata/merge_xx_current.xml create mode 100644 tools/localize/testdata/merge_xx_old.xml create mode 100644 tools/localize/testdata/pseudo.xliff create mode 100644 tools/localize/testdata/res/values-zz-rZZ/strings.xml create mode 100644 tools/localize/testdata/res/values/strings.xml create mode 100644 tools/localize/testdata/strip_xliff.xliff create mode 100644 tools/localize/testdata/values/strings.xml create mode 100644 tools/localize/testdata/xliff1.xliff create mode 100644 tools/localize/testdata/xml.xml create mode 100644 tools/localize/xmb.cpp create mode 100644 tools/localize/xmb.h (limited to 'tools/localize') diff --git a/tools/localize/Android.mk b/tools/localize/Android.mk new file mode 100644 index 0000000..186177f --- /dev/null +++ b/tools/localize/Android.mk @@ -0,0 +1,56 @@ +# +# Copyright 2006 The Android Open Source Project +# +# Android Asset Packaging Tool +# + +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := \ + file_utils.cpp \ + localize.cpp \ + merge_res_and_xliff.cpp \ + res_check.cpp \ + xmb.cpp \ + Configuration.cpp \ + Perforce.cpp \ + SourcePos.cpp \ + Values.cpp \ + ValuesFile.cpp \ + XLIFFFile.cpp \ + XMLHandler.cpp + +LOCAL_C_INCLUDES := \ + external/expat/lib \ + build/libs/host/include + +LOCAL_CFLAGS += -g -O0 + +LOCAL_STATIC_LIBRARIES := \ + libexpat \ + libhost \ + libutils \ + libcutils + +ifeq ($(HOST_OS),linux) +LOCAL_LDLIBS += -lrt +endif + + +LOCAL_MODULE := localize + +ifeq (a,a) + LOCAL_CFLAGS += -DLOCALIZE_WITH_TESTS + LOCAL_SRC_FILES += \ + test.cpp \ + localize_test.cpp \ + merge_res_and_xliff_test.cpp \ + Perforce_test.cpp \ + ValuesFile_test.cpp \ + XLIFFFile_test.cpp \ + XMLHandler_test.cpp +endif + +include $(BUILD_HOST_EXECUTABLE) + diff --git a/tools/localize/Configuration.cpp b/tools/localize/Configuration.cpp new file mode 100644 index 0000000..56addbd --- /dev/null +++ b/tools/localize/Configuration.cpp @@ -0,0 +1,76 @@ +#include "Configuration.h" +#include + +int +Configuration::Compare(const Configuration& that) const +{ + int n; + + n = locale.compare(that.locale); + if (n != 0) return n; + + n = vendor.compare(that.vendor); + if (n != 0) return n; + + n = orientation.compare(that.orientation); + if (n != 0) return n; + + n = density.compare(that.density); + if (n != 0) return n; + + n = touchscreen.compare(that.touchscreen); + if (n != 0) return n; + + n = keyboard.compare(that.keyboard); + if (n != 0) return n; + + n = navigation.compare(that.navigation); + if (n != 0) return n; + + n = screenSize.compare(that.screenSize); + if (n != 0) return n; + + return 0; +} + +string +Configuration::ToString() const +{ + string s; + if (locale.length() > 0) { + if (s.length() > 0) { + s += "-"; + } + s += locale; + } + return s; +} + +bool +split_locale(const string& in, string* language, string* region) +{ + const int len = in.length(); + if (len == 2) { + if (isalpha(in[0]) && isalpha(in[1])) { + *language = in; + region->clear(); + return true; + } else { + return false; + } + } + else if (len == 5) { + if (isalpha(in[0]) && isalpha(in[1]) && (in[2] == '_' || in[2] == '-') + && isalpha(in[3]) && isalpha(in[4])) { + language->assign(in.c_str(), 2); + region->assign(in.c_str()+3, 2); + return true; + } else { + return false; + } + } + else { + return false; + } +} + diff --git a/tools/localize/Configuration.h b/tools/localize/Configuration.h new file mode 100644 index 0000000..f91bf04 --- /dev/null +++ b/tools/localize/Configuration.h @@ -0,0 +1,38 @@ +#ifndef CONFIGURATION_H +#define CONFIGURATION_H + +#include + +using namespace std; + +struct Configuration +{ + string locale; + string vendor; + string orientation; + string density; + string touchscreen; + string keyboard; + string navigation; + string screenSize; + + // Compare two configurations + int Compare(const Configuration& that) const; + + inline bool operator<(const Configuration& that) const { return Compare(that) < 0; } + inline bool operator<=(const Configuration& that) const { return Compare(that) <= 0; } + inline bool operator==(const Configuration& that) const { return Compare(that) == 0; } + inline bool operator!=(const Configuration& that) const { return Compare(that) != 0; } + inline bool operator>=(const Configuration& that) const { return Compare(that) >= 0; } + inline bool operator>(const Configuration& that) const { return Compare(that) > 0; } + + // Parse a directory name, like "values-en-rUS". Return the first segment in resType. + bool ParseDiectoryName(const string& dir, string* resType); + + string ToString() const; +}; + +bool split_locale(const string& in, string* language, string* region); + + +#endif // CONFIGURATION_H diff --git a/tools/localize/Perforce.cpp b/tools/localize/Perforce.cpp new file mode 100644 index 0000000..3425668 --- /dev/null +++ b/tools/localize/Perforce.cpp @@ -0,0 +1,230 @@ +#include "Perforce.h" +#include "log.h" +#include +#include +#include +#include +#include +#include + +using namespace std; + +extern char** environ; + +int +Perforce::RunCommand(const string& cmd, string* result, bool printOnFailure) +{ + int err; + int outPipe[2]; + int errPipe[2]; + pid_t pid; + + log_printf("Perforce::RunCommand: %s\n", cmd.c_str()); + + err = pipe(outPipe); + err |= pipe(errPipe); + if (err == -1) { + printf("couldn't create pipe. exiting.\n"); + exit(1); + return -1; + } + + pid = fork(); + if (pid == -1) { + printf("couldn't fork. eixiting\n"); + exit(1); + return -1; + } + else if (pid == 0) { + char const* args[] = { + "/bin/sh", + "-c", + cmd.c_str(), + NULL + }; + close(outPipe[0]); + close(errPipe[0]); + dup2(outPipe[1], 1); + dup2(errPipe[1], 2); + execve(args[0], (char* const*)args, environ); + // done + } + + close(outPipe[1]); + close(errPipe[1]); + + result->clear(); + + char buf[1024]; + + // stdout + while (true) { + size_t amt = read(outPipe[0], buf, sizeof(buf)); + result->append(buf, amt); + if (amt <= 0) { + break; + } + } + + // stderr -- the messages are short so it ought to just fit in the buffer + string error; + while (true) { + size_t amt = read(errPipe[0], buf, sizeof(buf)); + error.append(buf, amt); + if (amt <= 0) { + break; + } + } + + close(outPipe[0]); + close(errPipe[0]); + + waitpid(pid, &err, 0); + if (WIFEXITED(err)) { + err = WEXITSTATUS(err); + } else { + err = -1; + } + if (err != 0 && printOnFailure) { + write(2, error.c_str(), error.length()); + } + return err; +} + +int +Perforce::GetResourceFileNames(const string& version, const string& base, + const vector& apps, vector* results, + bool printOnFailure) +{ + int err; + string text; + stringstream cmd; + + cmd << "p4 files"; + + const size_t I = apps.size(); + for (size_t i=0; i 1023) { + fprintf(stderr, "line too long!\n"); + return 1; + } + + string s(str, lineend-str); + + char filename[1024]; + char edit[1024]; + int count = sscanf(str, "%[^#]#%*d - %s change %*d %*[^\n]\n", filename, edit); + + if (count == 2 && 0 != strcmp("delete", edit)) { + results->push_back(string(filename)); + } + + str = lineend + 1; + } + + return err; +} + +int +Perforce::GetFile(const string& file, const string& version, string* result, + bool printOnFailure) +{ + stringstream cmd; + cmd << "p4 print -q \"" << file << '@' << version << '"'; + return RunCommand(cmd.str(), result, printOnFailure); +} + +string +Perforce::GetCurrentChange(bool printOnFailure) +{ + int err; + string text; + + err = RunCommand("p4 changes -m 1 \\#have", &text, printOnFailure); + if (err != 0) { + return ""; + } + + long long n; + int count = sscanf(text.c_str(), "Change %lld on", &n); + if (count != 1) { + return ""; + } + + char result[100]; + sprintf(result, "%lld", n); + + return string(result); +} + +static int +do_files(const string& op, const vector& files, bool printOnFailure) +{ + string text; + stringstream cmd; + + cmd << "p4 " << op; + + const size_t I = files.size(); + for (size_t i=0; i& files, bool printOnFailure) +{ + return do_files("edit", files, printOnFailure); +} + +int +Perforce::AddFiles(const vector& files, bool printOnFailure) +{ + return do_files("add", files, printOnFailure); +} + +int +Perforce::DeleteFiles(const vector& files, bool printOnFailure) +{ + return do_files("delete", files, printOnFailure); +} + +string +Perforce::Where(const string& depotPath, bool printOnFailure) +{ + int err; + string text; + string cmd = "p4 where "; + cmd += depotPath; + + err = RunCommand(cmd, &text, printOnFailure); + if (err != 0) { + return ""; + } + + size_t index = text.find(' '); + if (index == text.npos) { + return ""; + } + index = text.find(' ', index+1)+1; + if (index == text.npos) { + return ""; + } + + return text.substr(index, text.length()-index-1); +} + diff --git a/tools/localize/Perforce.h b/tools/localize/Perforce.h new file mode 100644 index 0000000..522797d --- /dev/null +++ b/tools/localize/Perforce.h @@ -0,0 +1,25 @@ +#ifndef PERFORCE_H +#define PERFORCE_H + +#include +#include + +using namespace std; + +class Perforce +{ +public: + static int RunCommand(const string& cmd, string* result, bool printOnFailure); + static int GetResourceFileNames(const string& version, const string& base, + const vector& apps, vector* result, + bool printOnFailure); + static int GetFile(const string& file, const string& version, string* result, + bool printOnFailure); + static string GetCurrentChange(bool printOnFailure); + static int EditFiles(const vector& filename, bool printOnFailure); + static int AddFiles(const vector& files, bool printOnFailure); + static int DeleteFiles(const vector& files, bool printOnFailure); + static string Where(const string& depotPath, bool printOnFailure); +}; + +#endif // PERFORCE_H diff --git a/tools/localize/Perforce_test.cpp b/tools/localize/Perforce_test.cpp new file mode 100644 index 0000000..142b20e --- /dev/null +++ b/tools/localize/Perforce_test.cpp @@ -0,0 +1,62 @@ +#include "Perforce.h" +#include + +static int +RunCommand_test() +{ + string result; + int err = Perforce::RunCommand("p4 help csommands", &result, true); + printf("err=%d result=[[%s]]\n", err, result.c_str()); + return 0; +} + +static int +GetResourceFileNames_test() +{ + vector results; + vector apps; + apps.push_back("apps/common"); + apps.push_back("apps/Contacts"); + int err = Perforce::GetResourceFileNames("43019", "//device", apps, &results, true); + if (err != 0) { + return err; + } + if (results.size() != 2) { + return 1; + } + if (results[0] != "//device/apps/common/res/values/strings.xml") { + return 1; + } + if (results[1] != "//device/apps/Contacts/res/values/strings.xml") { + return 1; + } + if (false) { + for (size_t i=0; i +#include + +using namespace std; + +const SourcePos GENERATED_POS("", -1); + +// ErrorPos +// ============================================================================= +struct ErrorPos +{ + string file; + int line; + string error; + + ErrorPos(); + ErrorPos(const ErrorPos& that); + ErrorPos(const string& file, int line, const string& error); + ~ErrorPos(); + bool operator<(const ErrorPos& rhs) const; + bool operator==(const ErrorPos& rhs) const; + ErrorPos& operator=(const ErrorPos& rhs); + + void Print(FILE* to) const; +}; + +static set g_errors; + +ErrorPos::ErrorPos() +{ +} + +ErrorPos::ErrorPos(const ErrorPos& that) + :file(that.file), + line(that.line), + error(that.error) +{ +} + +ErrorPos::ErrorPos(const string& f, int l, const string& e) + :file(f), + line(l), + error(e) +{ +} + +ErrorPos::~ErrorPos() +{ +} + +bool +ErrorPos::operator<(const ErrorPos& rhs) const +{ + if (this->file < rhs.file) return true; + if (this->file == rhs.file) { + if (this->line < rhs.line) return true; + if (this->line == rhs.line) { + if (this->error < rhs.error) return true; + } + } + return false; +} + +bool +ErrorPos::operator==(const ErrorPos& rhs) const +{ + return this->file == rhs.file + && this->line == rhs.line + && this->error == rhs.error; +} + +ErrorPos& +ErrorPos::operator=(const ErrorPos& rhs) +{ + this->file = rhs.file; + this->line = rhs.line; + this->error = rhs.error; + return *this; +} + +void +ErrorPos::Print(FILE* to) const +{ + if (this->line >= 0) { + fprintf(to, "%s:%d: %s\n", this->file.c_str(), this->line, this->error.c_str()); + } else { + fprintf(to, "%s: %s\n", this->file.c_str(), this->error.c_str()); + } +} + +// SourcePos +// ============================================================================= +SourcePos::SourcePos(const string& f, int l) + : file(f), line(l) +{ +} + +SourcePos::SourcePos(const SourcePos& that) + : file(that.file), line(that.line) +{ +} + +SourcePos::SourcePos() + : file("???", 0) +{ +} + +SourcePos::~SourcePos() +{ +} + +string +SourcePos::ToString() const +{ + char buf[1024]; + if (this->line >= 0) { + snprintf(buf, sizeof(buf)-1, "%s:%d", this->file.c_str(), this->line); + } else { + snprintf(buf, sizeof(buf)-1, "%s:", this->file.c_str()); + } + buf[sizeof(buf)-1] = '\0'; + return string(buf); +} + +int +SourcePos::Error(const char* fmt, ...) const +{ + int retval=0; + char buf[1024]; + va_list ap; + va_start(ap, fmt); + retval = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + char* p = buf + retval - 1; + while (p > buf && *p == '\n') { + *p = '\0'; + p--; + } + ErrorPos err(this->file, this->line, string(buf)); + if (g_errors.find(err) == g_errors.end()) { + err.Print(stderr); + g_errors.insert(err); + } + return retval; +} + +bool +SourcePos::HasErrors() +{ + return g_errors.size() > 0; +} + +void +SourcePos::PrintErrors(FILE* to) +{ + set::const_iterator it; + for (it=g_errors.begin(); it!=g_errors.end(); it++) { + it->Print(to); + } +} + + + + diff --git a/tools/localize/SourcePos.h b/tools/localize/SourcePos.h new file mode 100644 index 0000000..5027129 --- /dev/null +++ b/tools/localize/SourcePos.h @@ -0,0 +1,28 @@ +#ifndef SOURCEPOS_H +#define SOURCEPOS_H + +#include + +using namespace std; + +class SourcePos +{ +public: + string file; + int line; + + SourcePos(const string& f, int l); + SourcePos(const SourcePos& that); + SourcePos(); + ~SourcePos(); + + string ToString() const; + int Error(const char* fmt, ...) const; + + static bool HasErrors(); + static void PrintErrors(FILE* to); +}; + +extern const SourcePos GENERATED_POS; + +#endif // SOURCEPOS_H diff --git a/tools/localize/Values.cpp b/tools/localize/Values.cpp new file mode 100644 index 0000000..e396f8b --- /dev/null +++ b/tools/localize/Values.cpp @@ -0,0 +1,134 @@ +#include "Values.h" +#include + + +// ===================================================================================== +StringResource::StringResource(const SourcePos& p, const string& f, const Configuration& c, + const string& i, int ix, XMLNode* v, const int ve, const string& vs, + const string& cmnt) + :pos(p), + file(f), + config(c), + id(i), + index(ix), + value(v), + version(ve), + versionString(vs), + comment(cmnt) +{ +} + +StringResource::StringResource() + :pos(), + file(), + config(), + id(), + index(-1), + value(NULL), + version(), + versionString(), + comment() +{ +} + +StringResource::StringResource(const StringResource& that) + :pos(that.pos), + file(that.file), + config(that.config), + id(that.id), + index(that.index), + value(that.value), + version(that.version), + versionString(that.versionString), + comment(that.comment) +{ +} + +int +StringResource::Compare(const StringResource& that) const +{ + if (file != that.file) { + return file < that.file ? -1 : 1; + } + if (id != that.id) { + return id < that.id ? -1 : 1; + } + if (index != that.index) { + return index - that.index; + } + if (config != that.config) { + return config < that.config ? -1 : 1; + } + if (version != that.version) { + return version < that.version ? -1 : 1; + } + return 0; +} + +string +StringResource::TypedID() const +{ + string result; + if (index < 0) { + result = "string:"; + } else { + char n[20]; + sprintf(n, "%d:", index); + result = "array:"; + result += n; + } + result += id; + return result; +} + +static void +split(const string& raw, vector*parts) +{ + size_t index = 0; + while (true) { + size_t next = raw.find(':', index); + if (next != raw.npos) { + parts->push_back(string(raw, index, next-index)); + index = next + 1; + } else { + parts->push_back(string(raw, index)); + break; + } + } +} + +bool +StringResource::ParseTypedID(const string& raw, string* id, int* index) +{ + vector parts; + split(raw, &parts); + + const size_t N = parts.size(); + + for (size_t i=0; i + +using namespace std; + +enum { + CURRENT_VERSION, + OLD_VERSION +}; + +struct StringResource +{ + StringResource(); + StringResource(const SourcePos& pos, const string& file, const Configuration& config, + const string& id, int index, XMLNode* value, + int version, const string& versionString, const string& comment = ""); + StringResource(const StringResource& that); + + // Compare two configurations + int Compare(const StringResource& that) const; + + inline bool operator<(const StringResource& that) const { return Compare(that) < 0; } + inline bool operator<=(const StringResource& that) const { return Compare(that) <= 0; } + inline bool operator==(const StringResource& that) const { return Compare(that) == 0; } + inline bool operator!=(const StringResource& that) const { return Compare(that) != 0; } + inline bool operator>=(const StringResource& that) const { return Compare(that) >= 0; } + inline bool operator>(const StringResource& that) const { return Compare(that) > 0; } + + string TypedID() const; + static bool ParseTypedID(const string& typed, string* id, int* index); + + SourcePos pos; + string file; + Configuration config; + string id; + int index; + XMLNode* value; + int version; + string versionString; + string comment; +}; + +#endif // VALUES_H diff --git a/tools/localize/ValuesFile.cpp b/tools/localize/ValuesFile.cpp new file mode 100644 index 0000000..bd6f494 --- /dev/null +++ b/tools/localize/ValuesFile.cpp @@ -0,0 +1,266 @@ +#include "ValuesFile.h" + +#include "XMLHandler.h" + +#include +#include +#include +#include +#include + +using namespace std; + +const char* const ANDROID_XMLNS = "http://schemas.android.com/apk/res/android"; +const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2"; + +const char *const NS_MAP[] = { + "android", ANDROID_XMLNS, + "xliff", XLIFF_XMLNS, + NULL, NULL +}; + +const XMLNamespaceMap ANDROID_NAMESPACES(NS_MAP); + + +// ===================================================================================== +class ArrayHandler : public XMLHandler +{ +public: + ArrayHandler(ValuesFile* vf, int version, const string& versionString, const string& id); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + +private: + ValuesFile* m_vf; + int m_version; + int m_index; + string m_versionString; + string m_id; + string m_comment; +}; + +ArrayHandler::ArrayHandler(ValuesFile* vf, int version, const string& versionString, + const string& id) + :m_vf(vf), + m_version(version), + m_index(0), + m_versionString(versionString), + m_id(id) +{ +} + +int +ArrayHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next) +{ + if (ns == "" && name == "item") { + XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT); + m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(), + m_id, m_index, node, m_version, m_versionString, + trim_string(m_comment))); + *next = new NodeHandler(node, XMLNode::EXACT); + m_index++; + m_comment = ""; + return 0; + } else { + pos.Error("invalid <%s> element inside \n", name.c_str()); + return 1; + } +} + +int +ArrayHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +ArrayHandler::OnComment(const SourcePos& pos, const string& text) +{ + m_comment += text; + return 0; +} + +// ===================================================================================== +class ValuesHandler : public XMLHandler +{ +public: + ValuesHandler(ValuesFile* vf, int version, const string& versionString); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + +private: + ValuesFile* m_vf; + int m_version; + string m_versionString; + string m_comment; +}; + +ValuesHandler::ValuesHandler(ValuesFile* vf, int version, const string& versionString) + :m_vf(vf), + m_version(version), + m_versionString(versionString) +{ +} + +int +ValuesHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next) +{ + if (ns == "" && name == "string") { + string id = XMLAttribute::Find(attrs, "", "name", ""); + XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT); + m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(), + id, -1, node, m_version, m_versionString, + trim_string(m_comment))); + *next = new NodeHandler(node, XMLNode::EXACT); + } + else if (ns == "" && name == "array") { + string id = XMLAttribute::Find(attrs, "", "name", ""); + *next = new ArrayHandler(m_vf, m_version, m_versionString, id); + } + m_comment = ""; + return 0; +} + +int +ValuesHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +ValuesHandler::OnComment(const SourcePos& pos, const string& text) +{ + m_comment += text; + return 0; +} + +// ===================================================================================== +ValuesFile::ValuesFile(const Configuration& config) + :m_config(config), + m_strings(), + m_arrays() +{ +} + +ValuesFile::~ValuesFile() +{ +} + +ValuesFile* +ValuesFile::ParseFile(const string& filename, const Configuration& config, + int version, const string& versionString) +{ + ValuesFile* result = new ValuesFile(config); + + TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString)); + XMLHandler::ParseFile(filename, &top); + + return result; +} + +ValuesFile* +ValuesFile::ParseString(const string& filename, const string& text, const Configuration& config, + int version, const string& versionString) +{ + ValuesFile* result = new ValuesFile(config); + + TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString)); + XMLHandler::ParseString(filename, text, &top); + + return result; +} + +const Configuration& +ValuesFile::GetConfiguration() const +{ + return m_config; +} + +void +ValuesFile::AddString(const StringResource& str) +{ + if (str.index < 0) { + m_strings.insert(str); + } else { + m_arrays[str.id].insert(str); + } +} + +set +ValuesFile::GetStrings() const +{ + set result = m_strings; + + for (map >::const_iterator it = m_arrays.begin(); + it != m_arrays.end(); it++) { + result.insert(it->second.begin(), it->second.end()); + } + + return result; +} + +XMLNode* +ValuesFile::ToXMLNode() const +{ + XMLNode* root; + + // + { + vector attrs; + ANDROID_NAMESPACES.AddToAttributes(&attrs); + root = XMLNode::NewElement(GENERATED_POS, "", "resources", attrs, XMLNode::PRETTY); + } + + // + for (map >::const_iterator it = m_arrays.begin(); + it != m_arrays.end(); it++) { + vector arrayAttrs; + arrayAttrs.push_back(XMLAttribute("", "name", it->first)); + const set& items = it->second; + XMLNode* arrayNode = XMLNode::NewElement(items.begin()->pos, "", "array", arrayAttrs, + XMLNode::PRETTY); + root->EditChildren().push_back(arrayNode); + + // + for (set::const_iterator item = items.begin(); + item != items.end(); item++) { + XMLNode* itemNode = item->value->Clone(); + itemNode->SetName("", "item"); + itemNode->EditAttributes().clear(); + arrayNode->EditChildren().push_back(itemNode); + } + } + + // + for (set::const_iterator it=m_strings.begin(); it!=m_strings.end(); it++) { + const StringResource& str = *it; + vector attrs; + XMLNode* strNode = str.value->Clone(); + strNode->SetName("", "string"); + strNode->EditAttributes().clear(); + strNode->EditAttributes().push_back(XMLAttribute("", "name", str.id)); + root->EditChildren().push_back(strNode); + } + + return root; +} + +string +ValuesFile::ToString() const +{ + XMLNode* xml = ToXMLNode(); + string s = "\n"; + s += xml->ToString(ANDROID_NAMESPACES); + delete xml; + s += '\n'; + return s; +} + diff --git a/tools/localize/ValuesFile.h b/tools/localize/ValuesFile.h new file mode 100644 index 0000000..752fd78 --- /dev/null +++ b/tools/localize/ValuesFile.h @@ -0,0 +1,52 @@ +#ifndef VALUES_FILE_H +#define VALUES_FILE_H + +#include "SourcePos.h" +#include "Configuration.h" +#include "XMLHandler.h" +#include "Values.h" + +#include +#include + +using namespace std; + +extern const XMLNamespaceMap ANDROID_NAMESPACES; + +class ValuesFile +{ +public: + ValuesFile(const Configuration& config); + + static ValuesFile* ParseFile(const string& filename, const Configuration& config, + int version, const string& versionString); + static ValuesFile* ParseString(const string& filename, const string& text, + const Configuration& config, + int version, const string& versionString); + ~ValuesFile(); + + const Configuration& GetConfiguration() const; + + void AddString(const StringResource& str); + set GetStrings() const; + + // exports this file as a n XMLNode, you own this object + XMLNode* ToXMLNode() const; + + // writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of + // ToXMLNode()). + string ToString() const; + +private: + class ParseState; + friend class ValuesFile::ParseState; + friend class StringHandler; + + ValuesFile(); + + Configuration m_config; + set m_strings; + map > m_arrays; +}; + +#endif // VALUES_FILE_H diff --git a/tools/localize/ValuesFile_test.cpp b/tools/localize/ValuesFile_test.cpp new file mode 100644 index 0000000..56d2ec2 --- /dev/null +++ b/tools/localize/ValuesFile_test.cpp @@ -0,0 +1,54 @@ +#include "ValuesFile.h" +#include + +int +ValuesFile_test() +{ + int err = 0; + Configuration config; + config.locale = "zz_ZZ"; + ValuesFile* vf = ValuesFile::ParseFile("testdata/values/strings.xml", config, + OLD_VERSION, "1"); + + const set& strings = vf->GetStrings(); + string canonical = vf->ToString(); + + if (false) { + printf("Strings (%zd)\n", strings.size()); + for (set::const_iterator it=strings.begin(); + it!=strings.end(); it++) { + const StringResource& str = *it; + printf("%s: '%s'[%d]='%s' (%s) \n", str.pos.ToString().c_str(), + str.id.c_str(), str.index, + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.config.ToString().c_str(), str.comment.c_str()); + } + + printf("XML:[[%s]]\n", canonical.c_str()); + } + + const char * const EXPECTED = + "\n" + "\n" + " \n" + " Email\n" + " Home\n" + " Work\n" + " Other\\u2026\n" + " \n" + " Discard\n" + " abcd\n" + " abBbC\n" + "\n"; + + if (canonical != EXPECTED) { + fprintf(stderr, "ValuesFile_test failed\n"); + fprintf(stderr, "canonical=[[%s]]\n", canonical.c_str()); + fprintf(stderr, "EXPECTED=[[%s]]\n", EXPECTED); + err = 1; + } + + delete vf; + return err; +} diff --git a/tools/localize/XLIFFFile.cpp b/tools/localize/XLIFFFile.cpp new file mode 100644 index 0000000..51f81de --- /dev/null +++ b/tools/localize/XLIFFFile.cpp @@ -0,0 +1,609 @@ +#include "XLIFFFile.h" + +#include +#include +#include + +const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2"; + +const char *const NS_MAP[] = { + "", XLIFF_XMLNS, + "xml", XMLNS_XMLNS, + NULL, NULL +}; + +const XMLNamespaceMap XLIFF_NAMESPACES(NS_MAP); + +int +XLIFFFile::File::Compare(const XLIFFFile::File& that) const +{ + if (filename != that.filename) { + return filename < that.filename ? -1 : 1; + } + return 0; +} + +// ===================================================================================== +XLIFFFile::XLIFFFile() +{ +} + +XLIFFFile::~XLIFFFile() +{ +} + +static XMLNode* +get_unique_node(const XMLNode* parent, const string& ns, const string& name, bool required) +{ + size_t count = parent->CountElementsByName(ns, name); + if (count == 1) { + return parent->GetElementByNameAt(ns, name, 0); + } else { + if (required) { + SourcePos pos = count == 0 + ? parent->Position() + : parent->GetElementByNameAt(XLIFF_XMLNS, name, 1)->Position(); + pos.Error("<%s> elements must contain exactly one <%s> element", + parent->Name().c_str(), name.c_str()); + } + return NULL; + } +} + +XLIFFFile* +XLIFFFile::Parse(const string& filename) +{ + XLIFFFile* result = new XLIFFFile(); + + XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY); + if (root == NULL) { + return NULL; + } + + // + vector files = root->GetElementsByName(XLIFF_XMLNS, "file"); + for (size_t i=0; iGetAttribute("", "datatype", ""); + string originalFile = file->GetAttribute("", "original", ""); + + Configuration sourceConfig; + sourceConfig.locale = file->GetAttribute("", "source-language", ""); + result->m_sourceConfig = sourceConfig; + + Configuration targetConfig; + targetConfig.locale = file->GetAttribute("", "target-language", ""); + result->m_targetConfig = targetConfig; + + result->m_currentVersion = file->GetAttribute("", "build-num", ""); + result->m_oldVersion = "old"; + + // + XMLNode* body = get_unique_node(file, XLIFF_XMLNS, "body", true); + if (body == NULL) continue; + + // + vector transUnits = body->GetElementsByName(XLIFF_XMLNS, "trans-unit"); + for (size_t j=0; jGetAttribute("", "id", ""); + if (rawID == "") { + transUnit->Position().Error(" tag requires an id"); + continue; + } + string id; + int index; + + if (!StringResource::ParseTypedID(rawID, &id, &index)) { + transUnit->Position().Error(" has invalid id '%s'\n", rawID.c_str()); + continue; + } + + // + XMLNode* source = get_unique_node(transUnit, XLIFF_XMLNS, "source", false); + if (source != NULL) { + XMLNode* node = source->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(source->Position(), originalFile, + sourceConfig, id, index, node, CURRENT_VERSION, + result->m_currentVersion)); + } + + // + XMLNode* target = get_unique_node(transUnit, XLIFF_XMLNS, "target", false); + if (target != NULL) { + XMLNode* node = target->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(target->Position(), originalFile, + targetConfig, id, index, node, CURRENT_VERSION, + result->m_currentVersion)); + } + + // + XMLNode* altTrans = get_unique_node(transUnit, XLIFF_XMLNS, "alt-trans", false); + if (altTrans != NULL) { + // + XMLNode* altSource = get_unique_node(altTrans, XLIFF_XMLNS, "source", false); + if (altSource != NULL) { + XMLNode* node = altSource->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(altSource->Position(), + originalFile, sourceConfig, id, index, node, OLD_VERSION, + result->m_oldVersion)); + } + + // + XMLNode* altTarget = get_unique_node(altTrans, XLIFF_XMLNS, "target", false); + if (altTarget != NULL) { + XMLNode* node = altTarget->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(altTarget->Position(), + originalFile, targetConfig, id, index, node, OLD_VERSION, + result->m_oldVersion)); + } + } + } + } + delete root; + return result; +} + +XLIFFFile* +XLIFFFile::Create(const Configuration& sourceConfig, const Configuration& targetConfig, + const string& currentVersion) +{ + XLIFFFile* result = new XLIFFFile(); + result->m_sourceConfig = sourceConfig; + result->m_targetConfig = targetConfig; + result->m_currentVersion = currentVersion; + return result; +} + +set +XLIFFFile::Files() const +{ + set result; + for (vector::const_iterator f = m_files.begin(); f != m_files.end(); f++) { + result.insert(f->filename); + } + return result; +} + +void +XLIFFFile::AddStringResource(const StringResource& str) +{ + string id = str.TypedID(); + + File* f = NULL; + const size_t I = m_files.size(); + for (size_t i=0; itransUnits.size(); + TransUnit* g = NULL; + for (size_t j=0; jtransUnits[j].id == id) { + g = &f->transUnits[j]; + } + } + if (g == NULL) { + TransUnit group; + group.id = id; + f->transUnits.push_back(group); + g = &f->transUnits[J]; + } + + StringResource* res = find_string_res(*g, str); + if (res == NULL) { + return ; + } + if (res->id != "") { + str.pos.Error("Duplicate string resource: %s", res->id.c_str()); + res->pos.Error("Previous definition here"); + return ; + } + *res = str; + + m_strings.insert(str); +} + +void +XLIFFFile::Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie) +{ + const size_t I = m_files.size(); + for (size_t ix=0, i=I-1; ixType() == XMLNode::TEXT) { + addTo->EditChildren().push_back(original->Clone()); + return 0; + } else { + string ctype; + if (original->Namespace() == "") { + if (original->Name() == "b") { + ctype = "bold"; + } + else if (original->Name() == "i") { + ctype = "italic"; + } + else if (original->Name() == "u") { + ctype = "underline"; + } + } + if (ctype != "") { + vector attrs; + attrs.push_back(XMLAttribute(XLIFF_XMLNS, "ctype", ctype)); + XMLNode* copy = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, "g", + attrs, XMLNode::EXACT); + + const vector& children = original->Children(); + size_t I = children.size(); + for (size_t i=0; iNamespace() == XLIFF_XMLNS) { + addTo->EditChildren().push_back(original->Clone()); + return 0; + } else { + if (original->Namespace() == "") { + // flatten out the tag into ph tags -- but only if there is no namespace + // that's still unsupported because propagating the xmlns attribute is hard. + vector attrs; + char idStr[30]; + (*phID)++; + sprintf(idStr, "id-%d", *phID); + attrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", idStr)); + + if (original->Children().size() == 0) { + XMLNode* ph = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, + "ph", attrs, XMLNode::EXACT); + ph->EditChildren().push_back( + XMLNode::NewText(original->Position(), + original->ToString(XLIFF_NAMESPACES), + XMLNode::EXACT)); + addTo->EditChildren().push_back(ph); + } else { + XMLNode* begin = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, + "bpt", attrs, XMLNode::EXACT); + begin->EditChildren().push_back( + XMLNode::NewText(original->Position(), + original->OpenTagToString(XLIFF_NAMESPACES, XMLNode::EXACT), + XMLNode::EXACT)); + XMLNode* end = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, + "ept", attrs, XMLNode::EXACT); + string endText = "Name(); + endText += ">"; + end->EditChildren().push_back(XMLNode::NewText(original->Position(), + endText, XMLNode::EXACT)); + + addTo->EditChildren().push_back(begin); + + const vector& children = original->Children(); + size_t I = children.size(); + for (size_t i=0; iEditChildren().push_back(end); + } + return err; + } else { + original->Position().Error("invalid <%s> element in <%s> tag\n", + original->Name().c_str(), name.c_str()); + return 1; + } + } + } + } +} + +XMLNode* +create_string_node(const StringResource& str, const string& name) +{ + vector attrs; + attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve")); + XMLNode* node = XMLNode::NewElement(str.pos, XLIFF_XMLNS, name, attrs, XMLNode::EXACT); + + const vector& children = str.value->Children(); + size_t I = children.size(); + int err = 0; + for (size_t i=0; i + { + vector attrs; + XLIFF_NAMESPACES.AddToAttributes(&attrs); + attrs.push_back(XMLAttribute(XLIFF_XMLNS, "version", "1.2")); + root = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "xliff", attrs, XMLNode::PRETTY); + } + + vector groups; + + // + vector files = m_files; + sort(files.begin(), files.end()); + const size_t I = files.size(); + for (size_t i=0; i fileAttrs; + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "datatype", "x-android-res")); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "original", file.filename)); + + struct timeval tv; + struct timezone tz; + gettimeofday(&tv, &tz); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "date", trim_string(ctime(&tv.tv_sec)))); + + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "source-language", m_sourceConfig.locale)); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "target-language", m_targetConfig.locale)); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "build-num", m_currentVersion)); + + XMLNode* fileNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "file", fileAttrs, + XMLNode::PRETTY); + root->EditChildren().push_back(fileNode); + + // + XMLNode* bodyNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "body", + vector(), XMLNode::PRETTY); + fileNode->EditChildren().push_back(bodyNode); + + // + vector transUnits = file.transUnits; + sort(transUnits.begin(), transUnits.end(), compare_id); + const size_t J = transUnits.size(); + for (size_t j=0; j tuAttrs; + + // strings start with string: + tuAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", transUnit.id)); + XMLNode* transUnitNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "trans-unit", + tuAttrs, XMLNode::PRETTY); + bodyNode->EditChildren().push_back(transUnitNode); + + // + if (transUnit.source.comment != "") { + vector extradataAttrs; + XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "extradata", + extradataAttrs, XMLNode::EXACT); + transUnitNode->EditChildren().push_back(extraNode); + extraNode->EditChildren().push_back( + XMLNode::NewText(GENERATED_POS, transUnit.source.comment, + XMLNode::PRETTY)); + } + + // + if (transUnit.source.id != "") { + transUnitNode->EditChildren().push_back( + create_string_node(transUnit.source, "source")); + } + + // + if (transUnit.target.id != "") { + transUnitNode->EditChildren().push_back( + create_string_node(transUnit.target, "target")); + } + + // + if (transUnit.altSource.id != "" || transUnit.altTarget.id != "" + || transUnit.rejectComment != "") { + vector altTransAttrs; + XMLNode* altTransNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "alt-trans", + altTransAttrs, XMLNode::PRETTY); + transUnitNode->EditChildren().push_back(altTransNode); + + // + if (transUnit.rejectComment != "") { + vector extradataAttrs; + XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, + "extradata", extradataAttrs, + XMLNode::EXACT); + altTransNode->EditChildren().push_back(extraNode); + extraNode->EditChildren().push_back( + XMLNode::NewText(GENERATED_POS, transUnit.rejectComment, + XMLNode::PRETTY)); + } + + // + if (transUnit.altSource.id != "") { + altTransNode->EditChildren().push_back( + create_string_node(transUnit.altSource, "source")); + } + + // + if (transUnit.altTarget.id != "") { + altTransNode->EditChildren().push_back( + create_string_node(transUnit.altTarget, "target")); + } + } + + } + } + + return root; +} + + +string +XLIFFFile::ToString() const +{ + XMLNode* xml = ToXMLNode(); + string s = "\n"; + s += xml->ToString(XLIFF_NAMESPACES); + delete xml; + s += '\n'; + return s; +} + +Stats +XLIFFFile::GetStats(const string& config) const +{ + Stats stat; + stat.config = config; + stat.files = m_files.size(); + stat.toBeTranslated = 0; + stat.noComments = 0; + + for (vector::const_iterator file=m_files.begin(); file!=m_files.end(); file++) { + stat.toBeTranslated += file->transUnits.size(); + + for (vector::const_iterator tu=file->transUnits.begin(); + tu!=file->transUnits.end(); tu++) { + if (tu->source.comment == "") { + stat.noComments++; + } + } + } + + stat.totalStrings = stat.toBeTranslated; + + return stat; +} diff --git a/tools/localize/XLIFFFile.h b/tools/localize/XLIFFFile.h new file mode 100644 index 0000000..a93d479 --- /dev/null +++ b/tools/localize/XLIFFFile.h @@ -0,0 +1,98 @@ +#ifndef XLIFF_FILE_H +#define XLIFF_FILE_H + +#include "Values.h" + +#include "Configuration.h" + +#include + +using namespace std; + +extern const XMLNamespaceMap XLIFF_NAMESPACES; + +extern const char*const XLIFF_XMLNS; + +struct Stats +{ + string config; + size_t files; + size_t toBeTranslated; + size_t noComments; + size_t totalStrings; +}; + +struct TransUnit { + string id; + StringResource source; + StringResource target; + StringResource altSource; + StringResource altTarget; + string rejectComment; +}; + +class XLIFFFile +{ +public: + static XLIFFFile* Parse(const string& filename); + static XLIFFFile* Create(const Configuration& sourceConfig, const Configuration& targetConfig, + const string& currentVersion); + ~XLIFFFile(); + + inline const Configuration& SourceConfig() const { return m_sourceConfig; } + inline const Configuration& TargetConfig() const { return m_targetConfig; } + + inline const string& CurrentVersion() const { return m_currentVersion; } + inline const string& OldVersion() const { return m_oldVersion; } + + set Files() const; + + void AddStringResource(const StringResource& res); + inline set const& GetStringResources() const { return m_strings; } + bool FindStringResource(const string& filename, int version, bool source); + + void Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie); + void Map(void (*func)(const string&,TransUnit*,void*), void* cookie); + + TransUnit* EditTransUnit(const string& file, const string& id); + + // exports this file as a n XMLNode, you own this object + XMLNode* ToXMLNode() const; + + // writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of + // ToXMLNode()). + string ToString() const; + + Stats GetStats(const string& config) const; + +private: + struct File { + int Compare(const File& that) const; + + inline bool operator<(const File& that) const { return Compare(that) < 0; } + inline bool operator<=(const File& that) const { return Compare(that) <= 0; } + inline bool operator==(const File& that) const { return Compare(that) == 0; } + inline bool operator!=(const File& that) const { return Compare(that) != 0; } + inline bool operator>=(const File& that) const { return Compare(that) >= 0; } + inline bool operator>(const File& that) const { return Compare(that) > 0; } + + string filename; + vector transUnits; + }; + + XLIFFFile(); + StringResource* find_string_res(TransUnit& g, const StringResource& str); + + Configuration m_sourceConfig; + Configuration m_targetConfig; + + string m_currentVersion; + string m_oldVersion; + + set m_strings; + vector m_files; +}; + +int convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID); + +#endif // XLIFF_FILE_H diff --git a/tools/localize/XLIFFFile_test.cpp b/tools/localize/XLIFFFile_test.cpp new file mode 100644 index 0000000..52ed4c3 --- /dev/null +++ b/tools/localize/XLIFFFile_test.cpp @@ -0,0 +1,115 @@ +#include "XLIFFFile.h" +#include +#include "ValuesFile.h" + +XMLNode* create_string_node(const StringResource& str, const string& name); + +static int +Parse_test() +{ + XLIFFFile* xf = XLIFFFile::Parse("testdata/xliff1.xliff"); + if (xf == NULL) { + return 1; + } + + set const& strings = xf->GetStringResources(); + + if (false) { + for (set::iterator it=strings.begin(); it!=strings.end(); it++) { + const StringResource& str = *it; + printf("STRING!!! id=%s index=%d value='%s' pos=%s file=%s version=%d(%s)\n", + str.id.c_str(), str.index, + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.pos.ToString().c_str(), str.file.c_str(), str.version, + str.versionString.c_str()); + } + printf("XML:[[%s]]\n", xf->ToString().c_str()); + } + + delete xf; + return 0; +} + +static XMLNode* +add_html_tag(XMLNode* addTo, const string& tag) +{ + vector attrs; + XMLNode* node = XMLNode::NewElement(GENERATED_POS, "", tag, attrs, XMLNode::EXACT); + addTo->EditChildren().push_back(node); + return node; +} + +static int +create_string_node_test() +{ + int err = 0; + StringResource res; + vector attrs; + res.value = XMLNode::NewElement(GENERATED_POS, "", "something", attrs, XMLNode::EXACT); + res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " begin ", XMLNode::EXACT)); + + XMLNode* child; + + child = add_html_tag(res.value, "b"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT)); + + child = add_html_tag(res.value, "i"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "i", XMLNode::EXACT)); + + child = add_html_tag(child, "b"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT)); + + child = add_html_tag(res.value, "u"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "u", XMLNode::EXACT)); + + + res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " end ", XMLNode::EXACT)); + + XMLNode* xliff = create_string_node(res, "blah"); + + string oldString = res.value->ToString(XLIFF_NAMESPACES); + string newString = xliff->ToString(XLIFF_NAMESPACES); + + if (false) { + printf("OLD=\"%s\"\n", oldString.c_str()); + printf("NEW=\"%s\"\n", newString.c_str()); + } + + const char* const EXPECTED_OLD + = " begin bibu end "; + if (oldString != EXPECTED_OLD) { + fprintf(stderr, "oldString mismatch:\n"); + fprintf(stderr, " expected='%s'\n", EXPECTED_OLD); + fprintf(stderr, " actual='%s'\n", oldString.c_str()); + err |= 1; + } + + const char* const EXPECTED_NEW + = " begin b" + "ibu" + " end "; + if (newString != EXPECTED_NEW) { + fprintf(stderr, "newString mismatch:\n"); + fprintf(stderr, " expected='%s'\n", EXPECTED_NEW); + fprintf(stderr, " actual='%s'\n", newString.c_str()); + err |= 1; + } + + if (err != 0) { + fprintf(stderr, "create_string_node_test failed\n"); + } + return err; +} + +int +XLIFFFile_test() +{ + bool all = true; + int err = 0; + + if (all) err |= Parse_test(); + if (all) err |= create_string_node_test(); + + return err; +} + diff --git a/tools/localize/XMLHandler.cpp b/tools/localize/XMLHandler.cpp new file mode 100644 index 0000000..3fab211 --- /dev/null +++ b/tools/localize/XMLHandler.cpp @@ -0,0 +1,793 @@ +#include "XMLHandler.h" + +#include +#include +#include +#include +#include +#include +#include + +#define NS_SEPARATOR 1 +#define MORE_INDENT " " + +static string +xml_text_escape(const string& s) +{ + string result; + const size_t N = s.length(); + for (size_t i=0; i': + result += ">"; + break; + case '&': + result += "&"; + break; + default: + result += c; + break; + } + } + return result; +} + +static string +xml_attr_escape(const string& s) +{ + string result; + const size_t N = s.length(); + for (size_t i=0; i::const_iterator it = m_map.find(ns); + if (it == m_map.end()) { + return ""; + } else { + return it->second; + } +} + +string +XMLNamespaceMap::GetPrefix(const string& ns) const +{ + if (ns == "") { + return ""; + } + map::const_iterator it = m_map.find(ns); + if (it != m_map.end()) { + if (it->second == "") { + return ""; + } else { + return it->second + ":"; + } + } else { + return ":"; // invalid + } +} + +void +XMLNamespaceMap::AddToAttributes(vector* attrs) const +{ + map::const_iterator it; + for (it=m_map.begin(); it!=m_map.end(); it++) { + if (it->second == "xml") { + continue; + } + XMLAttribute attr; + if (it->second == "") { + attr.name = "xmlns"; + } else { + attr.name = "xmlns:"; + attr.name += it->second; + } + attr.value = it->first; + attrs->push_back(attr); + } +} + +XMLAttribute::XMLAttribute() +{ +} + +XMLAttribute::XMLAttribute(const XMLAttribute& that) + :ns(that.ns), + name(that.name), + value(that.value) +{ +} + +XMLAttribute::XMLAttribute(string n, string na, string v) + :ns(n), + name(na), + value(v) +{ +} + +XMLAttribute::~XMLAttribute() +{ +} + +int +XMLAttribute::Compare(const XMLAttribute& that) const +{ + if (ns != that.ns) { + return ns < that.ns ? -1 : 1; + } + if (name != that.name) { + return name < that.name ? -1 : 1; + } + return 0; +} + +string +XMLAttribute::Find(const vector& list, const string& ns, const string& name, + const string& def) +{ + const size_t N = list.size(); + for (size_t i=0; i stack; + XML_Parser parser; + vector*> attributes; + string filename; +}; + +XMLNode::XMLNode() +{ +} + +XMLNode::~XMLNode() +{ +// for_each(m_children.begin(), m_children.end(), delete_object); +} + +XMLNode* +XMLNode::Clone() const +{ + switch (m_type) { + case ELEMENT: { + XMLNode* e = XMLNode::NewElement(m_pos, m_ns, m_name, m_attrs, m_pretty); + const size_t N = m_children.size(); + for (size_t i=0; im_children.push_back(m_children[i]->Clone()); + } + return e; + } + case TEXT: { + return XMLNode::NewText(m_pos, m_text, m_pretty); + } + default: + return NULL; + } +} + +XMLNode* +XMLNode::NewElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, int pretty) +{ + XMLNode* node = new XMLNode(); + node->m_type = ELEMENT; + node->m_pretty = pretty; + node->m_pos = pos; + node->m_ns = ns; + node->m_name = name; + node->m_attrs = attrs; + return node; +} + +XMLNode* +XMLNode::NewText(const SourcePos& pos, const string& text, int pretty) +{ + XMLNode* node = new XMLNode(); + node->m_type = TEXT; + node->m_pretty = pretty; + node->m_pos = pos; + node->m_text = text; + return node; +} + +void +XMLNode::SetPrettyRecursive(int value) +{ + m_pretty = value; + const size_t N = m_children.size(); + for (size_t i=0; iSetPrettyRecursive(value); + } +} + +string +XMLNode::ContentsToString(const XMLNamespaceMap& nspaces) const +{ + return contents_to_string(nspaces, ""); +} + +string +XMLNode::ToString(const XMLNamespaceMap& nspaces) const +{ + return to_string(nspaces, ""); +} + +string +XMLNode::OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const +{ + return open_tag_to_string(nspaces, "", pretty); +} + +string +XMLNode::contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const +{ + string result; + const size_t N = m_children.size(); + for (size_t i=0; iType()) { + case ELEMENT: + if (m_pretty == PRETTY) { + result += '\n'; + result += indent; + } + case TEXT: + result += child->to_string(nspaces, indent); + break; + } + } + return result; +} + +string +trim_string(const string& str) +{ + const char* p = str.c_str(); + while (*p && isspace(*p)) { + p++; + } + const char* q = str.c_str() + str.length() - 1; + while (q > p && isspace(*q)) { + q--; + } + q++; + return string(p, q-p); +} + +string +XMLNode::open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent, int pretty) const +{ + if (m_type != ELEMENT) { + return ""; + } + string result = "<"; + result += nspaces.GetPrefix(m_ns); + result += m_name; + + vector attrs = m_attrs; + + sort(attrs.begin(), attrs.end()); + + const size_t N = attrs.size(); + for (size_t i=0; i 0) { + result += '>'; + } else { + result += " />"; + } + return result; +} + +string +XMLNode::to_string(const XMLNamespaceMap& nspaces, const string& indent) const +{ + switch (m_type) + { + case TEXT: { + if (m_pretty == EXACT) { + return xml_text_escape(m_text); + } else { + return xml_text_escape(trim_string(m_text)); + } + } + case ELEMENT: { + string result = open_tag_to_string(nspaces, indent, PRETTY); + + if (m_children.size() > 0) { + result += contents_to_string(nspaces, indent + MORE_INDENT); + + if (m_pretty == PRETTY && m_children.size() > 0) { + result += '\n'; + result += indent; + } + + result += "'; + } + return result; + } + default: + return ""; + } +} + +string +XMLNode::CollapseTextContents() const +{ + if (m_type == TEXT) { + return m_text; + } + else if (m_type == ELEMENT) { + string result; + + const size_t N=m_children.size(); + for (size_t i=0; iCollapseTextContents(); + } + + return result; + } + else { + return ""; + } +} + +vector +XMLNode::GetElementsByName(const string& ns, const string& name) const +{ + vector result; + const size_t N=m_children.size(); + for (size_t i=0; im_type == ELEMENT && child->m_ns == ns && child->m_name == name) { + result.push_back(child); + } + } + return result; +} + +XMLNode* +XMLNode::GetElementByNameAt(const string& ns, const string& name, size_t index) const +{ + vector result; + const size_t N=m_children.size(); + for (size_t i=0; im_type == ELEMENT && child->m_ns == ns && child->m_name == name) { + if (index == 0) { + return child; + } else { + index--; + } + } + } + return NULL; +} + +size_t +XMLNode::CountElementsByName(const string& ns, const string& name) const +{ + size_t result = 0; + const size_t N=m_children.size(); + for (size_t i=0; im_type == ELEMENT && child->m_ns == ns && child->m_name == name) { + result++; + } + } + return result; +} + +string +XMLNode::GetAttribute(const string& ns, const string& name, const string& def) const +{ + return XMLAttribute::Find(m_attrs, ns, name, def); +} + +static void +parse_namespace(const char* data, string* ns, string* name) +{ + const char* p = strchr(data, NS_SEPARATOR); + if (p != NULL) { + ns->assign(data, p-data); + name->assign(p+1); + } else { + ns->assign(""); + name->assign(data); + } +} + +static void +convert_attrs(const char** in, vector* out) +{ + while (*in) { + XMLAttribute attr; + parse_namespace(in[0], &attr.ns, &attr.name); + attr.value = in[1]; + out->push_back(attr); + in += 2; + } +} + +static bool +list_contains(const vector& stack, XMLHandler* handler) +{ + const size_t N = stack.size(); + for (size_t i=0; istack[data->stack.size()-1]; + + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + string nsString; + string nameString; + XMLHandler* next = handler; + vector attributes; + + parse_namespace(name, &nsString, &nameString); + convert_attrs(attrs, &attributes); + + handler->OnStartElement(pos, nsString, nameString, attributes, &next); + + if (next == NULL) { + next = handler; + } + + if (next != handler) { + next->elementPos = pos; + next->elementNamespace = nsString; + next->elementName = nameString; + next->elementAttributes = attributes; + } + + data->stack.push_back(next); +} + +static void XMLCALL +end_element_handler(void *userData, const char *name) +{ + xml_handler_data* data = (xml_handler_data*)userData; + + XMLHandler* handler = data->stack[data->stack.size()-1]; + data->stack.pop_back(); + + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + + if (!list_contains(data->stack, handler)) { + handler->OnDone(pos); + if (data->stack.size() > 1) { + // not top one + delete handler; + } + } + + handler = data->stack[data->stack.size()-1]; + + string nsString; + string nameString; + + parse_namespace(name, &nsString, &nameString); + + handler->OnEndElement(pos, nsString, nameString); +} + +static void XMLCALL +text_handler(void *userData, const XML_Char *s, int len) +{ + xml_handler_data* data = (xml_handler_data*)userData; + XMLHandler* handler = data->stack[data->stack.size()-1]; + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + handler->OnText(pos, string(s, len)); +} + +static void XMLCALL +comment_handler(void *userData, const char *comment) +{ + xml_handler_data* data = (xml_handler_data*)userData; + XMLHandler* handler = data->stack[data->stack.size()-1]; + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + handler->OnComment(pos, string(comment)); +} + +bool +XMLHandler::ParseFile(const string& filename, XMLHandler* handler) +{ + char buf[16384]; + int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) { + SourcePos(filename, -1).Error("Unable to open file for read: %s", strerror(errno)); + return false; + } + + XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR); + xml_handler_data state; + state.stack.push_back(handler); + state.parser = parser; + state.filename = filename; + + XML_SetUserData(parser, &state); + XML_SetElementHandler(parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler(parser, text_handler); + XML_SetCommentHandler(parser, comment_handler); + + ssize_t len; + bool done; + do { + len = read(fd, buf, sizeof(buf)); + done = len < (ssize_t)sizeof(buf); + if (len < 0) { + SourcePos(filename, -1).Error("Error reading file: %s\n", strerror(errno)); + close(fd); + return false; + } + if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) { + SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error( + "Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser))); + close(fd); + return false; + } + } while (!done); + + XML_ParserFree(parser); + + close(fd); + + return true; +} + +bool +XMLHandler::ParseString(const string& filename, const string& text, XMLHandler* handler) +{ + XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR); + xml_handler_data state; + state.stack.push_back(handler); + state.parser = parser; + state.filename = filename; + + XML_SetUserData(parser, &state); + XML_SetElementHandler(parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler(parser, text_handler); + XML_SetCommentHandler(parser, comment_handler); + + if (XML_Parse(parser, text.c_str(), text.size(), true) == XML_STATUS_ERROR) { + SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error( + "Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser))); + return false; + } + + XML_ParserFree(parser); + + return true; +} + +XMLHandler::XMLHandler() +{ +} + +XMLHandler::~XMLHandler() +{ +} + +int +XMLHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next) +{ + return 0; +} + +int +XMLHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name) +{ + return 0; +} + +int +XMLHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +XMLHandler::OnComment(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +XMLHandler::OnDone(const SourcePos& pos) +{ + return 0; +} + +TopElementHandler::TopElementHandler(const string& ns, const string& name, XMLHandler* next) + :m_ns(ns), + m_name(name), + m_next(next) +{ +} + +int +TopElementHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next) +{ + *next = m_next; + return 0; +} + +int +TopElementHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name) +{ + return 0; +} + +int +TopElementHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +TopElementHandler::OnDone(const SourcePos& pos) +{ + return 0; +} + + +NodeHandler::NodeHandler(XMLNode* root, int pretty) + :m_root(root), + m_pretty(pretty) +{ + if (root != NULL) { + m_nodes.push_back(root); + } +} + +NodeHandler::~NodeHandler() +{ +} + +int +NodeHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next) +{ + int pretty; + if (XMLAttribute::Find(attrs, XMLNS_XMLNS, "space", "") == "preserve") { + pretty = XMLNode::EXACT; + } else { + if (m_root == NULL) { + pretty = m_pretty; + } else { + pretty = m_nodes[m_nodes.size()-1]->Pretty(); + } + } + XMLNode* n = XMLNode::NewElement(pos, ns, name, attrs, pretty); + if (m_root == NULL) { + m_root = n; + } else { + m_nodes[m_nodes.size()-1]->EditChildren().push_back(n); + } + m_nodes.push_back(n); + return 0; +} + +int +NodeHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name) +{ + m_nodes.pop_back(); + return 0; +} + +int +NodeHandler::OnText(const SourcePos& pos, const string& text) +{ + if (m_root == NULL) { + return 1; + } + XMLNode* n = XMLNode::NewText(pos, text, m_nodes[m_nodes.size()-1]->Pretty()); + m_nodes[m_nodes.size()-1]->EditChildren().push_back(n); + return 0; +} + +int +NodeHandler::OnComment(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +NodeHandler::OnDone(const SourcePos& pos) +{ + return 0; +} + +XMLNode* +NodeHandler::ParseFile(const string& filename, int pretty) +{ + NodeHandler handler(NULL, pretty); + if (!XMLHandler::ParseFile(filename, &handler)) { + fprintf(stderr, "error parsing file: %s\n", filename.c_str()); + return NULL; + } + return handler.Root(); +} + +XMLNode* +NodeHandler::ParseString(const string& filename, const string& text, int pretty) +{ + NodeHandler handler(NULL, pretty); + if (!XMLHandler::ParseString(filename, text, &handler)) { + fprintf(stderr, "error parsing file: %s\n", filename.c_str()); + return NULL; + } + return handler.Root(); +} + + diff --git a/tools/localize/XMLHandler.h b/tools/localize/XMLHandler.h new file mode 100644 index 0000000..1130710 --- /dev/null +++ b/tools/localize/XMLHandler.h @@ -0,0 +1,197 @@ +#ifndef XML_H +#define XML_H + +#include "SourcePos.h" + +#include +#include +#include + +#define XMLNS_XMLNS "http://www.w3.org/XML/1998/namespace" + +using namespace std; + +string trim_string(const string& str); + +struct XMLAttribute +{ + string ns; + string name; + string value; + + XMLAttribute(); + XMLAttribute(const XMLAttribute& that); + XMLAttribute(string ns, string name, string value); + ~XMLAttribute(); + + int Compare(const XMLAttribute& that) const; + + inline bool operator<(const XMLAttribute& that) const { return Compare(that) < 0; } + inline bool operator<=(const XMLAttribute& that) const { return Compare(that) <= 0; } + inline bool operator==(const XMLAttribute& that) const { return Compare(that) == 0; } + inline bool operator!=(const XMLAttribute& that) const { return Compare(that) != 0; } + inline bool operator>=(const XMLAttribute& that) const { return Compare(that) >= 0; } + inline bool operator>(const XMLAttribute& that) const { return Compare(that) > 0; } + + static string Find(const vector& list, + const string& ns, const string& name, const string& def); +}; + +class XMLNamespaceMap +{ +public: + XMLNamespaceMap(); + XMLNamespaceMap(char const*const* nspaces); + string Get(const string& ns) const; + string GetPrefix(const string& ns) const; + void AddToAttributes(vector* attrs) const; +private: + map m_map; +}; + +struct XMLNode +{ +public: + enum { + EXACT = 0, + PRETTY = 1 + }; + + enum { + ELEMENT = 0, + TEXT = 1 + }; + + static XMLNode* NewElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, int pretty); + static XMLNode* NewText(const SourcePos& pos, const string& text, int pretty); + + ~XMLNode(); + + // a deep copy + XMLNode* Clone() const; + + inline int Type() const { return m_type; } + inline int Pretty() const { return m_pretty; } + void SetPrettyRecursive(int value); + string ContentsToString(const XMLNamespaceMap& nspaces) const; + string ToString(const XMLNamespaceMap& nspaces) const; + string OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const; + + string CollapseTextContents() const; + + inline const SourcePos& Position() const { return m_pos; } + + // element + inline string Namespace() const { return m_ns; } + inline string Name() const { return m_name; } + inline void SetName(const string& ns, const string& n) { m_ns = ns; m_name = n; } + inline const vector& Attributes() const { return m_attrs; } + inline vector& EditAttributes() { return m_attrs; } + inline const vector& Children() const { return m_children; } + inline vector& EditChildren() { return m_children; } + vector GetElementsByName(const string& ns, const string& name) const; + XMLNode* GetElementByNameAt(const string& ns, const string& name, size_t index) const; + size_t CountElementsByName(const string& ns, const string& name) const; + string GetAttribute(const string& ns, const string& name, const string& def) const; + + // text + inline string Text() const { return m_text; } + +private: + XMLNode(); + XMLNode(const XMLNode&); + + string contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const; + string to_string(const XMLNamespaceMap& nspaces, const string& indent) const; + string open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent, + int pretty) const; + + int m_type; + int m_pretty; + SourcePos m_pos; + + // element + string m_ns; + string m_name; + vector m_attrs; + vector m_children; + + // text + string m_text; +}; + +class XMLHandler +{ +public: + // information about the element that started us + SourcePos elementPos; + string elementNamespace; + string elementName; + vector elementAttributes; + + XMLHandler(); + virtual ~XMLHandler(); + + XMLHandler* parent; + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next); + virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + virtual int OnDone(const SourcePos& pos); + + static bool ParseFile(const string& filename, XMLHandler* handler); + static bool ParseString(const string& filename, const string& text, XMLHandler* handler); +}; + +class TopElementHandler : public XMLHandler +{ +public: + TopElementHandler(const string& ns, const string& name, XMLHandler* next); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next); + virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnDone(const SourcePos& endPos); + +private: + string m_ns; + string m_name; + XMLHandler* m_next; +}; + +class NodeHandler : public XMLHandler +{ +public: + // after it's done, you own everything created and added to root + NodeHandler(XMLNode* root, int pretty); + ~NodeHandler(); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector& attrs, XMLHandler** next); + virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + virtual int OnDone(const SourcePos& endPos); + + inline XMLNode* Root() const { return m_root; } + + static XMLNode* ParseFile(const string& filename, int pretty); + static XMLNode* ParseString(const string& filename, const string& text, int pretty); + +private: + XMLNode* m_root; + int m_pretty; + vector m_nodes; +}; + +template +static void delete_object(T* obj) +{ + delete obj; +} + +#endif // XML_H diff --git a/tools/localize/XMLHandler_test.cpp b/tools/localize/XMLHandler_test.cpp new file mode 100644 index 0000000..1c81c0c --- /dev/null +++ b/tools/localize/XMLHandler_test.cpp @@ -0,0 +1,133 @@ +#include "XMLHandler.h" +#include +#include +#include + +const char *const NS_MAP[] = { + "xml", XMLNS_XMLNS, + NULL, NULL +}; + +const XMLNamespaceMap NO_NAMESPACES(NS_MAP); + +char const*const EXPECTED_EXACT = + "\n" + " \n" + " \n" + " \n" + " asdf\n" + " \n" + " \n" + " a,b \n" + " a,b \n" + " \n" + "\n"; + +char const*const EXPECTED_PRETTY = + "\n" + " \n" + " \n" + " \n" + " asdf\n" + " \n" + " \n" + " a\n" + " ,\n" + " b \n" + " \n" + " a,b \n" + " \n" + "\n"; + +static string +read_file(const string& filename) +{ + char buf[1024]; + int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) { + return ""; + } + string result; + while (true) { + ssize_t len = read(fd, buf, sizeof(buf)-1); + buf[len] = '\0'; + if (len <= 0) { + break; + } + result.append(buf, len); + } + close(fd); + return result; +} + +static int +ParseFile_EXACT_test() +{ + XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::EXACT); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_EXACT; +} + +static int +ParseFile_PRETTY_test() +{ + XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::PRETTY); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_PRETTY; +} + +static int +ParseString_EXACT_test() +{ + string text = read_file("testdata/xml.xml"); + XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::EXACT); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_EXACT; +} + +static int +ParseString_PRETTY_test() +{ + string text = read_file("testdata/xml.xml"); + XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::PRETTY); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_PRETTY; +} + +int +XMLHandler_test() +{ + int err = 0; + bool all = true; + + if (all) err |= ParseFile_EXACT_test(); + if (all) err |= ParseFile_PRETTY_test(); + if (all) err |= ParseString_EXACT_test(); + if (all) err |= ParseString_PRETTY_test(); + + return err; +} diff --git a/tools/localize/XMLNode.h b/tools/localize/XMLNode.h new file mode 100644 index 0000000..bfb9f55 --- /dev/null +++ b/tools/localize/XMLNode.h @@ -0,0 +1,19 @@ +#ifndef XMLNODE_H +#define XMLNODE_H + +#include + +using namespace std; + +struct XMLAttribute +{ + string ns; + string name; + string value; + + static string Find(const vector& list, + const string& ns, const string& name, const string& def); +}; + + +#endif // XMLNODE_H diff --git a/tools/localize/file_utils.cpp b/tools/localize/file_utils.cpp new file mode 100644 index 0000000..bb82a9c --- /dev/null +++ b/tools/localize/file_utils.cpp @@ -0,0 +1,143 @@ +#include +#include +#include +#include "file_utils.h" +#include "Perforce.h" +#include +#include +#include +#include +#include "log.h" + +string +translated_file_name(const string& file, const string& locale) +{ + const char* str = file.c_str(); + const char* p = str + file.length(); + const char* rest = NULL; + const char* values = p; + + while (p > str) { + p--; + if (*p == '/') { + rest = values; + values = p; + if (0 == strncmp("values", values+1, rest-values-1)) { + break; + } + } + } + values++; + + string result(str, values-str); + result.append(values, rest-values); + + string language, region; + if (locale == "") { + language = ""; + region = ""; + } + else if (!split_locale(locale, &language, ®ion)) { + return ""; + } + + if (language != "") { + result += '-'; + result += language; + } + if (region != "") { + result += "-r"; + result += region; + } + + result += rest; + + return result; +} + +ValuesFile* +get_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure) +{ + int err; + string text; + + log_printf("get_values_file filename=%s\n", filename.c_str()); + err = Perforce::GetFile(filename, versionString, &text, printOnFailure); + if (err != 0 || text == "") { + return NULL; + } + + ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version, + versionString); + if (result == NULL) { + fprintf(stderr, "unable to parse file: %s\n", filename.c_str()); + exit(1); + } + return result; +} + +ValuesFile* +get_local_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure) +{ + int err; + string text; + char buf[2049]; + int fd; + ssize_t amt; + + fd = open(filename.c_str(), O_RDONLY); + if (fd == -1) { + fprintf(stderr, "unable to open file: %s\n", filename.c_str()); + return NULL; + } + + while ((amt = read(fd, buf, sizeof(buf)-1)) > 0) { + text.append(buf, amt); + } + + close(fd); + + if (text == "") { + return NULL; + } + + ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version, + versionString); + if (result == NULL) { + fprintf(stderr, "unable to parse file: %s\n", filename.c_str()); + exit(1); + } + return result; +} + +void +print_file_status(size_t j, size_t J, const string& message) +{ + printf("\r%s file %zd of %zd...", message.c_str(), j, J); + fflush(stdout); +} + +int +write_to_file(const string& filename, const string& text) +{ + mkdirs(parent_dir(filename).c_str()); + int fd = open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); + if (fd < 0) { + fprintf(stderr, "unable to open file for write (%s): %s\n", strerror(errno), + filename.c_str()); + return -1; + } + + ssize_t amt = write(fd, text.c_str(), text.length()); + + close(fd); + + if (amt < 0) { + return amt; + } + return amt == (ssize_t)text.length() ? 0 : -1; +} + + diff --git a/tools/localize/file_utils.h b/tools/localize/file_utils.h new file mode 100644 index 0000000..3b3fa21 --- /dev/null +++ b/tools/localize/file_utils.h @@ -0,0 +1,21 @@ +#ifndef FILE_UTILS_H +#define FILE_UTILS_H + +#include "ValuesFile.h" +#include "Configuration.h" +#include + +using namespace std; + +string translated_file_name(const string& file, const string& locale); + +ValuesFile* get_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure); +ValuesFile* get_local_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure); + +void print_file_status(size_t j, size_t J, const string& message = "Reading"); +int write_to_file(const string& filename, const string& text); + + +#endif // FILE_UTILS_H diff --git a/tools/localize/localize.cpp b/tools/localize/localize.cpp new file mode 100644 index 0000000..c0d84cc --- /dev/null +++ b/tools/localize/localize.cpp @@ -0,0 +1,767 @@ +#include "SourcePos.h" +#include "ValuesFile.h" +#include "XLIFFFile.h" +#include "Perforce.h" +#include "merge_res_and_xliff.h" +#include "localize.h" +#include "file_utils.h" +#include "res_check.h" +#include "xmb.h" + +#include + +#include +#include +#include +#include +#include + +using namespace std; + +FILE* g_logFile = NULL; + +int test(); + +int +read_settings(const string& filename, map* result, const string& rootDir) +{ + XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY); + if (root == NULL) { + SourcePos(filename, -1).Error("Error reading file."); + return 1; + } + + // + vector configNodes = root->GetElementsByName("", "configuration"); + const size_t I = configNodes.size(); + for (size_t i=0; iGetAttribute("", "id", ""); + if (settings.id == "") { + configNode->Position().Error(" needs an id attribute."); + delete root; + return 1; + } + + settings.oldVersion = configNode->GetAttribute("", "old-cl", ""); + + settings.currentVersion = configNode->GetAttribute("", "new-cl", ""); + if (settings.currentVersion == "") { + configNode->Position().Error(" needs a new-cl attribute."); + delete root; + return 1; + } + + // + vector appNodes = configNode->GetElementsByName("", "app"); + + const size_t J = appNodes.size(); + for (size_t j=0; jGetAttribute("", "dir", ""); + if (dir == "") { + appNode->Position().Error(" needs a dir attribute."); + delete root; + return 1; + } + + settings.apps.push_back(dir); + } + + // + vector rejectNodes = configNode->GetElementsByName("", "reject"); + + const size_t K = rejectNodes.size(); + for (size_t k=0; kGetAttribute("", "file", ""); + if (reject.file == "") { + rejectNode->Position().Error(" needs a file attribute."); + delete root; + return 1; + } + string f = reject.file; + reject.file = rootDir; + reject.file += '/'; + reject.file += f; + + reject.name = rejectNode->GetAttribute("", "name", ""); + if (reject.name == "") { + rejectNode->Position().Error(" needs a name attribute."); + delete root; + return 1; + } + + reject.comment = trim_string(rejectNode->CollapseTextContents()); + + settings.reject.push_back(reject); + } + + (*result)[settings.id] = settings; + } + + delete root; + return 0; +} + + +static void +ValuesFile_to_XLIFFFile(const ValuesFile* values, XLIFFFile* xliff, const string& englishFilename) +{ + const set& strings = values->GetStrings(); + for (set::const_iterator it=strings.begin(); it!=strings.end(); it++) { + StringResource res = *it; + res.file = englishFilename; + xliff->AddStringResource(res); + } +} + +static bool +contains_reject(const Settings& settings, const string& file, const TransUnit& tu) +{ + const string name = tu.id; + const vector& reject = settings.reject; + const size_t I = reject.size(); + for (size_t i=0; i field themselves. Or if the string hasn't changed since last time, we can just + * not even tell them about it. As the project nears the end, it will be convenient to see + * the xliff files reducing in size, so we pick the latter. Obviously, if the string has + * changed, then we need to get it retranslated. + */ +bool +keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie) +{ + const Settings* settings = reinterpret_cast(cookie); + + if (contains_reject(*settings, file, unit)) { + return true; + } + + if (unit.source.id == "") { + return false; + } + if (unit.altTarget.id == "" || unit.altSource.id == "") { + return true; + } + return unit.source.value->ContentsToString(XLIFF_NAMESPACES) + != unit.altSource.value->ContentsToString(XLIFF_NAMESPACES); +} + +int +validate_config(const string& settingsFile, const map& settings, + const string& config) +{ + if (settings.find(config) == settings.end()) { + SourcePos(settingsFile, -1).Error("settings file does not contain setting: %s\n", + config.c_str()); + return 1; + } + return 0; +} + +int +validate_configs(const string& settingsFile, const map& settings, + const vector& configs) +{ + int err = 0; + for (size_t i=0; i *resFiles, const string& config, + const map& settings, const string& rootDir) +{ + int err; + vector > allResFiles; + vector configs; + configs.push_back(config); + err = select_files(&allResFiles, configs, settings, rootDir); + if (err == 0) { + *resFiles = allResFiles[0]; + } + return err; +} + +int +select_files(vector > *allResFiles, const vector& configs, + const map& settings, const string& rootDir) +{ + int err; + printf("Selecting files..."); + fflush(stdout); + + for (size_t i=0; isecond; + + vector resFiles; + err = Perforce::GetResourceFileNames(setting.currentVersion, rootDir, + setting.apps, &resFiles, true); + if (err != 0) { + fprintf(stderr, "error with perforce. bailing\n"); + return err; + } + + allResFiles->push_back(resFiles); + } + return 0; +} + +static int +do_export(const string& settingsFile, const string& rootDir, const string& outDir, + const string& targetLocale, const vector& configs) +{ + bool success = true; + int err; + + if (false) { + printf("settingsFile=%s\n", settingsFile.c_str()); + printf("rootDir=%s\n", rootDir.c_str()); + printf("outDir=%s\n", outDir.c_str()); + for (size_t i=0; i settings; + err = read_settings(settingsFile, &settings, rootDir); + if (err != 0) { + return err; + } + + err = validate_configs(settingsFile, settings, configs); + if (err != 0) { + return err; + } + + vector > allResFiles; + err = select_files(&allResFiles, configs, settings, rootDir); + if (err != 0) { + return err; + } + + size_t totalFileCount = 0; + for (size_t i=0; i stats; + vector > xliffs; + + for (size_t i=0; i& resFiles = allResFiles[i]; + const size_t J = resFiles.size(); + for (size_t j=0; jToString().c_str()); + } else { + fprintf(stderr, "error reading file %s@%s\n", resFile.c_str(), + setting.currentVersion.c_str()); + success = false; + } + + // old file + print_file_status(++fileProgress, totalFileCount); + ValuesFile* oldFile = get_values_file(resFile, english, OLD_VERSION, + setting.oldVersion, false); + if (oldFile != NULL) { + ValuesFile_to_XLIFFFile(oldFile, xliff, resFile); + //printf("oldFile=[%s]\n", oldFile->ToString().c_str()); + } + + // translated version + // (get the head of the tree for the most recent translation, but it's considered + // the old one because the "current" one hasn't been made yet, and this goes into + // the tag if necessary + print_file_status(++fileProgress, totalFileCount); + string transFilename = translated_file_name(resFile, targetLocale); + ValuesFile* transFile = get_values_file(transFilename, translated, OLD_VERSION, + setting.currentVersion, false); + if (transFile != NULL) { + ValuesFile_to_XLIFFFile(transFile, xliff, resFile); + } + + delete currentFile; + delete oldFile; + delete transFile; + } + + Stats beforeFilterStats = xliff->GetStats(config); + + // run through the XLIFFFile and strip out TransUnits that have identical + // old and current source values and are not in the reject list, or just + // old values and no source values + xliff->Filter(keep_this_trans_unit, (void*)&setting); + + Stats afterFilterStats = xliff->GetStats(config); + afterFilterStats.totalStrings = beforeFilterStats.totalStrings; + + // add the reject comments + for (vector::const_iterator reject = setting.reject.begin(); + reject != setting.reject.end(); reject++) { + TransUnit* tu = xliff->EditTransUnit(reject->file, reject->name); + tu->rejectComment = reject->comment; + } + + // config-locale-current_cl.xliff + stringstream filename; + if (outDir != "") { + filename << outDir << '/'; + } + filename << config << '-' << targetLocale << '-' << setting.currentVersion << ".xliff"; + xliffs.push_back(pair(filename.str(), xliff)); + + stats.push_back(afterFilterStats); + } + + // today is a good day to die + if (!success || SourcePos::HasErrors()) { + return 1; + } + + // write the XLIFF files + printf("\nWriting %zd file%s...\n", xliffs.size(), xliffs.size() == 1 ? "" : "s"); + for (vector >::iterator it = xliffs.begin(); it != xliffs.end(); it++) { + const string& filename = it->first; + XLIFFFile* xliff = it->second; + string text = xliff->ToString(); + write_to_file(filename, text); + } + + // the stats + printf("\n" + " to without total\n" + " config files translate comments strings\n" + "-----------------------------------------------------------------------\n"); + Stats totals; + totals.config = "total"; + totals.files = 0; + totals.toBeTranslated = 0; + totals.noComments = 0; + totals.totalStrings = 0; + for (vector::iterator it=stats.begin(); it!=stats.end(); it++) { + string cfg = it->config; + if (cfg.length() > 20) { + cfg.resize(20); + } + printf(" %-20s %-9zd %-9zd %-9zd %-19zd\n", cfg.c_str(), it->files, + it->toBeTranslated, it->noComments, it->totalStrings); + totals.files += it->files; + totals.toBeTranslated += it->toBeTranslated; + totals.noComments += it->noComments; + totals.totalStrings += it->totalStrings; + } + if (stats.size() > 1) { + printf("-----------------------------------------------------------------------\n" + " %-20s %-9zd %-9zd %-9zd %-19zd\n", totals.config.c_str(), totals.files, + totals.toBeTranslated, totals.noComments, totals.totalStrings); + } + printf("\n"); + return 0; +} + +struct PseudolocalizeSettings { + XLIFFFile* xliff; + bool expand; +}; + + +string +pseudolocalize_string(const string& source, const PseudolocalizeSettings* settings) +{ + return pseudolocalize_string(source); +} + +static XMLNode* +pseudolocalize_xml_node(const XMLNode* source, const PseudolocalizeSettings* settings) +{ + if (source->Type() == XMLNode::TEXT) { + return XMLNode::NewText(source->Position(), pseudolocalize_string(source->Text(), settings), + source->Pretty()); + } else { + XMLNode* target; + if (source->Namespace() == XLIFF_XMLNS && source->Name() == "g") { + // XXX don't translate these + target = XMLNode::NewElement(source->Position(), source->Namespace(), + source->Name(), source->Attributes(), source->Pretty()); + } else { + target = XMLNode::NewElement(source->Position(), source->Namespace(), + source->Name(), source->Attributes(), source->Pretty()); + } + + const vector& children = source->Children(); + const size_t I = children.size(); + for (size_t i=0; iEditChildren().push_back(pseudolocalize_xml_node(children[i], settings)); + } + + return target; + } +} + +void +pseudolocalize_trans_unit(const string&file, TransUnit* unit, void* cookie) +{ + const PseudolocalizeSettings* settings = (PseudolocalizeSettings*)cookie; + + const StringResource& source = unit->source; + StringResource* target = &unit->target; + *target = source; + + target->config = settings->xliff->TargetConfig(); + + delete target->value; + target->value = pseudolocalize_xml_node(source.value, settings); +} + +int +pseudolocalize_xliff(XLIFFFile* xliff, bool expand) +{ + PseudolocalizeSettings settings; + + settings.xliff = xliff; + settings.expand = expand; + xliff->Map(pseudolocalize_trans_unit, &settings); + return 0; +} + +static int +do_pseudo(const string& infile, const string& outfile, bool expand) +{ + int err; + + XLIFFFile* xliff = XLIFFFile::Parse(infile); + if (xliff == NULL) { + return 1; + } + + pseudolocalize_xliff(xliff, expand); + + err = write_to_file(outfile, xliff->ToString()); + + delete xliff; + + return err; +} + +void +log_printf(const char *fmt, ...) +{ + int ret; + va_list ap; + + if (g_logFile != NULL) { + va_start(ap, fmt); + ret = vfprintf(g_logFile, fmt, ap); + va_end(ap); + fflush(g_logFile); + } +} + +void +close_log_file() +{ + if (g_logFile != NULL) { + fclose(g_logFile); + } +} + +void +open_log_file(const char* file) +{ + g_logFile = fopen(file, "w"); + printf("log file: %s -- %p\n", file, g_logFile); + atexit(close_log_file); +} + +static int +usage() +{ + fprintf(stderr, + "usage: localize export OPTIONS CONFIGS...\n" + " REQUIRED OPTIONS\n" + " --settings SETTINGS The settings file to use. See CONFIGS below.\n" + " --root TREE_ROOT The location in Perforce of the files. e.g. //device\n" + " --target LOCALE The target locale. See LOCALES below.\n" + "\n" + " OPTIONAL OPTIONS\n" + " --out DIR Directory to put the output files. Defaults to the\n" + " current directory if not supplied. Files are\n" + " named as follows:\n" + " CONFIG-LOCALE-CURRENT_CL.xliff\n" + "\n" + "\n" + "usage: localize import XLIFF_FILE...\n" + "\n" + "Import a translated XLIFF file back into the tree.\n" + "\n" + "\n" + "usage: localize xlb XMB_FILE VALUES_FILES...\n" + "\n" + "Read resource files from the tree file and write the corresponding XLB file\n" + "\n" + "Supply all of the android resource files (values files) to export after that.\n" + "\n" + "\n" + "\n" + "CONFIGS\n" + "\n" + "LOCALES\n" + "Locales are specified in the form en_US They will be processed correctly\n" + "to locate the resouce files in the tree.\n" + "\n" + "\n" + "usage: localize pseudo OPTIONS INFILE [OUTFILE]\n" + " OPTIONAL OPTIONS\n" + " --big Pad strings so they get longer.\n" + "\n" + "Read INFILE, an XLIFF file, and output a pseudotranslated version of that file. If\n" + "OUTFILE is specified, the results are written there; otherwise, the results are\n" + "written back to INFILE.\n" + "\n" + "\n" + "usage: localize rescheck FILES...\n" + "\n" + "Reads the base strings and prints warnings about bad resources from the given files.\n" + "\n"); + return 1; +} + +int +main(int argc, const char** argv) +{ + //open_log_file("log.txt"); + //g_logFile = stdout; + + if (argc == 2 && 0 == strcmp(argv[1], "--test")) { + return test(); + } + + if (argc < 2) { + return usage(); + } + + int index = 1; + + if (0 == strcmp("export", argv[index])) { + string settingsFile; + string rootDir; + string outDir; + string baseLocale = "en"; + string targetLocale; + string language, region; + vector configs; + + index++; + while (index < argc) { + if (0 == strcmp("--settings", argv[index])) { + settingsFile = argv[index+1]; + index += 2; + } + else if (0 == strcmp("--root", argv[index])) { + rootDir = argv[index+1]; + index += 2; + } + else if (0 == strcmp("--out", argv[index])) { + outDir = argv[index+1]; + index += 2; + } + else if (0 == strcmp("--target", argv[index])) { + targetLocale = argv[index+1]; + index += 2; + } + else if (argv[index][0] == '-') { + fprintf(stderr, "unknown argument %s\n", argv[index]); + return usage(); + } + else { + break; + } + } + for (; index xliffFilenames; + + index++; + for (; index resFiles; + + index++; + if (argc < index+1) { + return usage(); + } + + outfile = argv[index]; + + index++; + for (; index files; + + index++; + while (index < argc) { + if (argv[index][0] == '-') { + fprintf(stderr, "unknown argument %s\n", argv[index]); + return usage(); + } + else { + break; + } + } + for (; index +#include + +using namespace std; + +struct Reject +{ + string file; + string name; + string comment; +}; + +struct Settings +{ + string id; + string oldVersion; + string currentVersion; + vector apps; + vector reject; +}; + +int read_settings(const string& filename, map* result, const string& rootDir); +string translated_file_name(const string& file, const string& locale); +bool keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie); +int validate_config(const string& settingsFile, const map& settings, + const string& configs); +int validate_configs(const string& settingsFile, const map& settings, + const vector& configs); +int select_files(vector *resFiles, const string& config, + const map& settings, const string& rootDir); +int select_files(vector > *allResFiles, const vector& configs, + const map& settings, const string& rootDir); + + +#endif // LOCALIZE_H diff --git a/tools/localize/localize_test.cpp b/tools/localize/localize_test.cpp new file mode 100644 index 0000000..63d904c --- /dev/null +++ b/tools/localize/localize_test.cpp @@ -0,0 +1,219 @@ +#include "XLIFFFile.h" +#include "ValuesFile.h" +#include "localize.h" + +int pseudolocalize_xliff(XLIFFFile* xliff, bool expand); + +static int +test_filename(const string& file, const string& locale, const string& expected) +{ + string result = translated_file_name(file, locale); + if (result != expected) { + fprintf(stderr, "translated_file_name test failed\n"); + fprintf(stderr, " locale='%s'\n", locale.c_str()); + fprintf(stderr, " expected='%s'\n", expected.c_str()); + fprintf(stderr, " result='%s'\n", result.c_str()); + return 1; + } else { + if (false) { + fprintf(stderr, "translated_file_name test passed\n"); + fprintf(stderr, " locale='%s'\n", locale.c_str()); + fprintf(stderr, " expected='%s'\n", expected.c_str()); + fprintf(stderr, " result='%s'\n", result.c_str()); + } + return 0; + } +} + +static int +translated_file_name_test() +{ + bool all = true; + int err = 0; + + if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz_ZZ", + "//device/samples/NotePad/res/values-zz-rZZ/strings.xml"); + + if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz", + "//device/samples/NotePad/res/values-zz/strings.xml"); + + if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "", + "//device/samples/NotePad/res/values/strings.xml"); + + return err; +} + +bool +return_false(const string&, const TransUnit& unit, void* cookie) +{ + return false; +} + +static int +delete_trans_units() +{ + XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff"); + if (xliff == NULL) { + printf("couldn't read file\n"); + return 1; + } + if (false) { + printf("XLIFF was [[%s]]\n", xliff->ToString().c_str()); + } + + xliff->Filter(return_false, NULL); + + if (false) { + printf("XLIFF is [[%s]]\n", xliff->ToString().c_str()); + + set const& strings = xliff->GetStringResources(); + printf("strings.size=%zd\n", strings.size()); + for (set::iterator it=strings.begin(); it!=strings.end(); it++) { + const StringResource& str = *it; + printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(), + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.pos.ToString().c_str(), str.file.c_str(), str.version, + str.versionString.c_str()); + } + } + + return 0; +} + +static int +filter_trans_units() +{ + XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff"); + if (xliff == NULL) { + printf("couldn't read file\n"); + return 1; + } + + if (false) { + printf("XLIFF was [[%s]]\n", xliff->ToString().c_str()); + } + + Settings setting; + xliff->Filter(keep_this_trans_unit, &setting); + + if (false) { + printf("XLIFF is [[%s]]\n", xliff->ToString().c_str()); + + set const& strings = xliff->GetStringResources(); + printf("strings.size=%zd\n", strings.size()); + for (set::iterator it=strings.begin(); it!=strings.end(); it++) { + const StringResource& str = *it; + printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(), + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.pos.ToString().c_str(), str.file.c_str(), str.version, + str.versionString.c_str()); + } + } + + return 0; +} + +static int +settings_test() +{ + int err; + map settings; + map::iterator it; + + err = read_settings("testdata/config.xml", &settings, "//asdf"); + if (err != 0) { + return err; + } + + if (false) { + for (it=settings.begin(); it!=settings.end(); it++) { + const Settings& setting = it->second; + printf("CONFIG:\n"); + printf(" id='%s'\n", setting.id.c_str()); + printf(" oldVersion='%s'\n", setting.oldVersion.c_str()); + printf(" currentVersion='%s'\n", setting.currentVersion.c_str()); + int i=0; + for (vector::const_iterator app=setting.apps.begin(); + app!=setting.apps.end(); app++) { + printf(" apps[%02d]='%s'\n", i, app->c_str()); + i++; + } + i=0; + for (vector::const_iterator reject=setting.reject.begin(); + reject!=setting.reject.end(); reject++) { + i++; + printf(" reject[%02d]=('%s','%s','%s')\n", i, reject->file.c_str(), + reject->name.c_str(), reject->comment.c_str()); + } + } + } + + for (it=settings.begin(); it!=settings.end(); it++) { + const Settings& setting = it->second; + if (it->first != setting.id) { + fprintf(stderr, "it->first='%s' setting.id='%s'\n", it->first.c_str(), + setting.id.c_str()); + err |= 1; + } + } + + + return err; +} + +static int +test_one_pseudo(bool big, const char* expected) +{ + XLIFFFile* xliff = XLIFFFile::Parse("testdata/pseudo.xliff"); + if (xliff == NULL) { + printf("couldn't read file\n"); + return 1; + } + if (false) { + printf("XLIFF was [[%s]]\n", xliff->ToString().c_str()); + } + + pseudolocalize_xliff(xliff, big); + string newString = xliff->ToString(); + delete xliff; + + if (false) { + printf("XLIFF is [[%s]]\n", newString.c_str()); + } + + if (false && newString != expected) { + fprintf(stderr, "xliff didn't translate as expected\n"); + fprintf(stderr, "newString=[[%s]]\n", newString.c_str()); + fprintf(stderr, "expected=[[%s]]\n", expected); + return 1; + } + + return 0; +} + +static int +pseudolocalize_test() +{ + int err = 0; + + err |= test_one_pseudo(false, ""); + //err |= test_one_pseudo(true, ""); + + return err; +} + +int +localize_test() +{ + bool all = true; + int err = 0; + + if (all) err |= translated_file_name_test(); + if (all) err |= delete_trans_units(); + if (all) err |= filter_trans_units(); + if (all) err |= settings_test(); + if (all) err |= pseudolocalize_test(); + + return err; +} + diff --git a/tools/localize/log.h b/tools/localize/log.h new file mode 100644 index 0000000..4a5fa7f --- /dev/null +++ b/tools/localize/log.h @@ -0,0 +1,7 @@ +#ifndef LOG_H +#define LOG_H + +void log_printf(const char* fmt, ...); + +#endif // LOG_H + diff --git a/tools/localize/merge_res_and_xliff.cpp b/tools/localize/merge_res_and_xliff.cpp new file mode 100644 index 0000000..58a6554 --- /dev/null +++ b/tools/localize/merge_res_and_xliff.cpp @@ -0,0 +1,391 @@ +#include "merge_res_and_xliff.h" + +#include "file_utils.h" +#include "Perforce.h" +#include "log.h" + +static set::const_iterator +find_id(const set& s, const string& id, int index) +{ + for (set::const_iterator it = s.begin(); it != s.end(); it++) { + if (it->id == id && it->index == index) { + return it; + } + } + return s.end(); +} + +static set::const_iterator +find_in_xliff(const set& s, const string& filename, const string& id, int index, + int version, const Configuration& config) +{ + for (set::const_iterator it = s.begin(); it != s.end(); it++) { + if (it->file == filename && it->id == id && it->index == index && it->version == version + && it->config == config) { + return it; + } + } + return s.end(); +} + + +static void +printit(const set& s, const set::const_iterator& it) +{ + if (it == s.end()) { + printf("(none)\n"); + } else { + printf("id=%s index=%d config=%s file=%s value='%s'\n", it->id.c_str(), it->index, + it->config.ToString().c_str(), it->file.c_str(), + it->value->ToString(ANDROID_NAMESPACES).c_str()); + } +} + +StringResource +convert_resource(const StringResource& s, const string& file, const Configuration& config, + int version, const string& versionString) +{ + return StringResource(s.pos, file, config, s.id, s.index, s.value ? s.value->Clone() : NULL, + version, versionString, s.comment); +} + +static bool +resource_has_contents(const StringResource& res) +{ + XMLNode* value = res.value; + if (value == NULL) { + return false; + } + string contents = value->ContentsToString(ANDROID_NAMESPACES); + return contents != ""; +} + +ValuesFile* +merge_res_and_xliff(const ValuesFile* en_currentFile, + const ValuesFile* xx_currentFile, const ValuesFile* xx_oldFile, + const string& filename, const XLIFFFile* xliffFile) +{ + bool success = true; + + Configuration en_config = xliffFile->SourceConfig(); + Configuration xx_config = xliffFile->TargetConfig(); + string currentVersion = xliffFile->CurrentVersion(); + + ValuesFile* result = new ValuesFile(xx_config); + + set en_cur = en_currentFile->GetStrings(); + set xx_cur = xx_currentFile->GetStrings(); + set xx_old = xx_oldFile->GetStrings(); + set xliff = xliffFile->GetStringResources(); + + // for each string in en_current + for (set::const_iterator en_c = en_cur.begin(); + en_c != en_cur.end(); en_c++) { + set::const_iterator xx_c = find_id(xx_cur, en_c->id, en_c->index); + set::const_iterator xx_o = find_id(xx_old, en_c->id, en_c->index); + set::const_iterator xlf = find_in_xliff(xliff, en_c->file, en_c->id, + en_c->index, CURRENT_VERSION, xx_config); + + if (false) { + printf("\nen_c: "); printit(en_cur, en_c); + printf("xx_c: "); printit(xx_cur, xx_c); + printf("xx_o: "); printit(xx_old, xx_o); + printf("xlf: "); printit(xliff, xlf); + } + + // if it changed between xx_old and xx_current, use xx_current + // (someone changed it by hand) + if (xx_o != xx_old.end() && xx_c != xx_cur.end()) { + string xx_o_value = xx_o->value->ToString(ANDROID_NAMESPACES); + string xx_c_value = xx_c->value->ToString(ANDROID_NAMESPACES); + if (xx_o_value != xx_c_value && xx_c_value != "") { + StringResource r(convert_resource(*xx_c, filename, xx_config, + CURRENT_VERSION, currentVersion)); + if (resource_has_contents(r)) { + result->AddString(r); + } + continue; + } + } + + // if it is present in xliff, use that + // (it just got translated) + if (xlf != xliff.end() && xlf->value->ToString(ANDROID_NAMESPACES) != "") { + StringResource r(convert_resource(*xlf, filename, xx_config, + CURRENT_VERSION, currentVersion)); + if (resource_has_contents(r)) { + result->AddString(r); + } + } + + // if it is present in xx_current, use that + // (it was already translated, and not retranslated) + // don't filter out empty strings if they were added by hand, the above code just + // guarantees that this tool never adds an empty one. + if (xx_c != xx_cur.end()) { + StringResource r(convert_resource(*xx_c, filename, xx_config, + CURRENT_VERSION, currentVersion)); + result->AddString(r); + } + + // othwerwise, leave it out. The resource fall-through code will use the English + // one at runtime, and the xliff export code will pick it up for translation next time. + } + + if (success) { + return result; + } else { + delete result; + return NULL; + } +} + + +struct MergedFile { + XLIFFFile* xliff; + string xliffFilename; + string original; + string translated; + ValuesFile* en_current; + ValuesFile* xx_current; + ValuesFile* xx_old; + ValuesFile* xx_new; + string xx_new_text; + string xx_new_filename; + bool new_file; + bool deleted_file; + + MergedFile(); + MergedFile(const MergedFile&); +}; + +struct compare_filenames { + bool operator()(const MergedFile& lhs, const MergedFile& rhs) const + { + return lhs.original < rhs.original; + } +}; + +MergedFile::MergedFile() + :xliff(NULL), + xliffFilename(), + original(), + translated(), + en_current(NULL), + xx_current(NULL), + xx_old(NULL), + xx_new(NULL), + xx_new_text(), + xx_new_filename(), + new_file(false), + deleted_file(false) +{ +} + +MergedFile::MergedFile(const MergedFile& that) + :xliff(that.xliff), + xliffFilename(that.xliffFilename), + original(that.original), + translated(that.translated), + en_current(that.en_current), + xx_current(that.xx_current), + xx_old(that.xx_old), + xx_new(that.xx_new), + xx_new_text(that.xx_new_text), + xx_new_filename(that.xx_new_filename), + new_file(that.new_file), + deleted_file(that.deleted_file) +{ +} + + +typedef set MergedFileSet; + +int +do_merge(const vector& xliffFilenames) +{ + int err = 0; + MergedFileSet files; + + printf("\rPreparing..."); fflush(stdout); + string currentChange = Perforce::GetCurrentChange(true); + + // for each xliff, make a MergedFile record and do a little error checking + for (vector::const_iterator xliffFilename=xliffFilenames.begin(); + xliffFilename!=xliffFilenames.end(); xliffFilename++) { + XLIFFFile* xliff = XLIFFFile::Parse(*xliffFilename); + if (xliff == NULL) { + fprintf(stderr, "localize import: unable to read file %s\n", xliffFilename->c_str()); + err = 1; + continue; + } + + set xf = xliff->Files(); + for (set::const_iterator f=xf.begin(); f!=xf.end(); f++) { + MergedFile mf; + mf.xliff = xliff; + mf.xliffFilename = *xliffFilename; + mf.original = *f; + mf.translated = translated_file_name(mf.original, xliff->TargetConfig().locale); + log_printf("mf.translated=%s mf.original=%s locale=%s\n", mf.translated.c_str(), + mf.original.c_str(), xliff->TargetConfig().locale.c_str()); + + if (files.find(mf) != files.end()) { + fprintf(stderr, "%s: duplicate string resources for file %s\n", + xliffFilename->c_str(), f->c_str()); + fprintf(stderr, "%s: previously defined here.\n", + files.find(mf)->xliffFilename.c_str()); + err = 1; + continue; + } + files.insert(mf); + } + } + + size_t deletedFileCount = 0; + size_t J = files.size() * 3; + size_t j = 1; + // Read all of the files from perforce. + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast(&(*mf)); + // file->en_current + print_file_status(j++, J); + file->en_current = get_values_file(file->original, file->xliff->SourceConfig(), + CURRENT_VERSION, currentChange, true); + if (file->en_current == NULL) { + // deleted file + file->deleted_file = true; + deletedFileCount++; + continue; + } + + // file->xx_current; + print_file_status(j++, J); + file->xx_current = get_values_file(file->translated, file->xliff->TargetConfig(), + CURRENT_VERSION, currentChange, false); + if (file->xx_current == NULL) { + file->xx_current = new ValuesFile(file->xliff->TargetConfig()); + file->new_file = true; + } + + // file->xx_old (note that the xliff's current version is our old version, because that + // was the current version when it was exported) + print_file_status(j++, J); + file->xx_old = get_values_file(file->translated, file->xliff->TargetConfig(), + OLD_VERSION, file->xliff->CurrentVersion(), false); + if (file->xx_old == NULL) { + file->xx_old = new ValuesFile(file->xliff->TargetConfig()); + file->new_file = true; + } + } + + // merge them + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast(&(*mf)); + if (file->deleted_file) { + continue; + } + file->xx_new = merge_res_and_xliff(file->en_current, file->xx_current, file->xx_old, + file->original, file->xliff); + } + + // now is a good time to stop if there was an error + if (err != 0) { + return err; + } + + // locate the files + j = 1; + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast(&(*mf)); + print_file_status(j++, J, "Locating"); + + file->xx_new_filename = Perforce::Where(file->translated, true); + if (file->xx_new_filename == "") { + fprintf(stderr, "\nWas not able to determine the location of depot file %s\n", + file->translated.c_str()); + err = 1; + } + } + + if (err != 0) { + return err; + } + + // p4 edit the files + // only do this if it changed - no need to submit files that haven't changed meaningfully + vector filesToEdit; + vector filesToAdd; + vector filesToDelete; + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast(&(*mf)); + if (file->deleted_file) { + filesToDelete.push_back(file->xx_new_filename); + continue; + } + string xx_current_text = file->xx_current->ToString(); + string xx_new_text = file->xx_new->ToString(); + if (xx_current_text != xx_new_text) { + if (file->xx_new->GetStrings().size() == 0) { + file->deleted_file = true; + filesToDelete.push_back(file->xx_new_filename); + } else { + file->xx_new_text = xx_new_text; + if (file->new_file) { + filesToAdd.push_back(file->xx_new_filename); + } else { + filesToEdit.push_back(file->xx_new_filename); + } + } + } + } + if (filesToAdd.size() == 0 && filesToEdit.size() == 0 && deletedFileCount == 0) { + printf("\nAll of the files are the same. Nothing to change.\n"); + return 0; + } + if (filesToEdit.size() > 0) { + printf("\np4 editing files...\n"); + if (0 != Perforce::EditFiles(filesToEdit, true)) { + return 1; + } + } + + + printf("\n"); + + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast(&(*mf)); + if (file->deleted_file) { + continue; + } + if (file->xx_new_text != "" && file->xx_new_filename != "") { + if (0 != write_to_file(file->xx_new_filename, file->xx_new_text)) { + err = 1; + } + } + } + + if (err != 0) { + return err; + } + + if (filesToAdd.size() > 0) { + printf("p4 adding %zd new files...\n", filesToAdd.size()); + err = Perforce::AddFiles(filesToAdd, true); + } + + if (filesToDelete.size() > 0) { + printf("p4 deleting %zd removed files...\n", filesToDelete.size()); + err = Perforce::DeleteFiles(filesToDelete, true); + } + + if (err != 0) { + return err; + } + + printf("\n" + "Theoretically, this merge was successfull. Next you should\n" + "review the diffs, get a code review, and submit it. Enjoy.\n\n"); + return 0; +} + diff --git a/tools/localize/merge_res_and_xliff.h b/tools/localize/merge_res_and_xliff.h new file mode 100644 index 0000000..acf2fff --- /dev/null +++ b/tools/localize/merge_res_and_xliff.h @@ -0,0 +1,13 @@ +#ifndef MERGE_RES_AND_XLIFF_H +#define MERGE_RES_AND_XLIFF_H + +#include "ValuesFile.h" +#include "XLIFFFile.h" + +ValuesFile* merge_res_and_xliff(const ValuesFile* en_current, + const ValuesFile* xx_current, const ValuesFile* xx_old, + const string& filename, const XLIFFFile* xliff); + +int do_merge(const vector& xliffFilenames); + +#endif // MERGE_RES_AND_XLIFF_H diff --git a/tools/localize/merge_res_and_xliff_test.cpp b/tools/localize/merge_res_and_xliff_test.cpp new file mode 100644 index 0000000..5a2b0f4 --- /dev/null +++ b/tools/localize/merge_res_and_xliff_test.cpp @@ -0,0 +1,47 @@ +#include "merge_res_and_xliff.h" + + +int +merge_test() +{ + Configuration english; + english.locale = "en_US"; + Configuration translated; + translated.locale = "zz_ZZ"; + + ValuesFile* en_current = ValuesFile::ParseFile("testdata/merge_en_current.xml", english, + CURRENT_VERSION, "3"); + if (en_current == NULL) { + fprintf(stderr, "merge_test: unable to read testdata/merge_en_current.xml\n"); + return 1; + } + + ValuesFile* xx_current = ValuesFile::ParseFile("testdata/merge_xx_current.xml", translated, + CURRENT_VERSION, "3"); + if (xx_current == NULL) { + fprintf(stderr, "merge_test: unable to read testdata/merge_xx_current.xml\n"); + return 1; + } + ValuesFile* xx_old = ValuesFile::ParseFile("testdata/merge_xx_old.xml", translated, + OLD_VERSION, "2"); + if (xx_old == NULL) { + fprintf(stderr, "merge_test: unable to read testdata/merge_xx_old.xml\n"); + return 1; + } + + XLIFFFile* xliff = XLIFFFile::Parse("testdata/merge.xliff"); + + ValuesFile* result = merge_res_and_xliff(en_current, xx_current, xx_old, + "//device/tools/localize/testdata/res/values/strings.xml", xliff); + + if (result == NULL) { + fprintf(stderr, "merge_test: result is NULL\n"); + return 1; + } + + printf("======= RESULT =======\n%s===============\n", result->ToString().c_str()); + + return 0; +} + + diff --git a/tools/localize/res_check.cpp b/tools/localize/res_check.cpp new file mode 100644 index 0000000..0fab98a --- /dev/null +++ b/tools/localize/res_check.cpp @@ -0,0 +1,106 @@ +#include "res_check.h" +#include "localize.h" +#include "file_utils.h" +#include "ValuesFile.h" + +#include + +static int check_file(const ValuesFile* file); +static int check_value(const SourcePos& pos, const XMLNode* value); +static int scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth = 0); + +int +do_rescheck(const vector& files) +{ + int err; + + Configuration english; + english.locale = "en_US"; + + for (size_t i=0; i strings = file->GetStrings(); + for (set::iterator it=strings.begin(); it!=strings.end(); it++) { + XMLNode* value = it->value; + if (value != NULL) { + err |= check_value(it->pos, value); + } + } + return err; +} + +static bool +contains_percent(const string& str) +{ + const size_t len = str.length(); + for (size_t i=0; iType() == XMLNode::ELEMENT) { + int err = 0; + if (depth == 0 || !is_xliff_block(value->Namespace(), value->Name())) { + const vector& children = value->Children(); + for (size_t i=0; iText()); + } +} + diff --git a/tools/localize/res_check.h b/tools/localize/res_check.h new file mode 100644 index 0000000..86e7ce6 --- /dev/null +++ b/tools/localize/res_check.h @@ -0,0 +1,12 @@ +#ifndef RESCHECK_H +#define RESCHECK_H + +#include +#include +#include + +using namespace std; + +int do_rescheck(const vector& files); + +#endif // RESCHECK_H diff --git a/tools/localize/test.cpp b/tools/localize/test.cpp new file mode 100644 index 0000000..5fa2c17 --- /dev/null +++ b/tools/localize/test.cpp @@ -0,0 +1,31 @@ +#include "SourcePos.h" +#include + +int ValuesFile_test(); +int XLIFFFile_test(); +int XMLHandler_test(); +int Perforce_test(); +int localize_test(); +int merge_test(); + +int +test() +{ + bool all = true; + int err = 0; + + if (all) err |= XMLHandler_test(); + if (all) err |= ValuesFile_test(); + if (all) err |= XLIFFFile_test(); + if (all) err |= Perforce_test(); + if (all) err |= localize_test(); + if (all) err |= merge_test(); + + if (err != 0) { + fprintf(stderr, "some tests failed\n"); + } else { + fprintf(stderr, "all tests passed\n"); + } + + return err; +} diff --git a/tools/localize/testdata/config.xml b/tools/localize/testdata/config.xml new file mode 100644 index 0000000..affa140 --- /dev/null +++ b/tools/localize/testdata/config.xml @@ -0,0 +1,15 @@ + + + + + + + + QA says this sounds rude. + + + diff --git a/tools/localize/testdata/import.xliff b/tools/localize/testdata/import.xliff new file mode 100644 index 0000000..b99b739 --- /dev/null +++ b/tools/localize/testdata/import.xliff @@ -0,0 +1,72 @@ + + + + + + aaa + AAA + + + bbb + BBB + + + ddd + DDDD + + + 1-One + 1-oNE + + + 1-Two + 1-tWO + + + 1-Three + 1-tHREE + + + 2-One + 2-oNE + + + 2-Two + 2-tWO + + + 2-Three + 2-tHREE + + + 2-Four + 2-fOUR + + + 4-One + 4-oNE + + + 4-Two + 4-tWO + + + 4-Three + 4-tHREE + + + + + + + diff --git a/tools/localize/testdata/merge.xliff b/tools/localize/testdata/merge.xliff new file mode 100644 index 0000000..2b78c45 --- /dev/null +++ b/tools/localize/testdata/merge.xliff @@ -0,0 +1,72 @@ + + + + + + aaa + AAA + + + bbb + BBB + + + ddd + DDDD + + + 1-One + 1-oNE + + + 1-Two + 1-tWO + + + 1-Three + 1-tHREE + + + 2-One + 2-oNE + + + 2-Two + 2-tWO + + + 2-Three + 2-tHREE + + + 2-Four + 2-fOUR + + + 4-One + 4-oNE + + + 4-Two + 4-tWO + + + 4-Three + 4-tHREE + + + + + + + diff --git a/tools/localize/testdata/merge_en_current.xml b/tools/localize/testdata/merge_en_current.xml new file mode 100644 index 0000000..6a11e68 --- /dev/null +++ b/tools/localize/testdata/merge_en_current.xml @@ -0,0 +1,44 @@ + + + + + aaa + bbb + ccc + ccc + + bolditalicitalic_underlineunderline + + + + 1-One + 1-Two + 1-Three + 1-Four + + + + 2-One + 2-Two + 2-Three + + + + 3-One + 3-Two + 3-Three + + diff --git a/tools/localize/testdata/merge_en_old.xml b/tools/localize/testdata/merge_en_old.xml new file mode 100644 index 0000000..933f98e --- /dev/null +++ b/tools/localize/testdata/merge_en_old.xml @@ -0,0 +1,45 @@ + + + + + aaa + bbb + ccc + ddd + + bolditalicitalic_underlineunderline + + + + 1-One + 1-Two + 1-Three + + + + 2-One + 2-Two + 2-Three + 2-Four + + + + 4-One + 4-Two + 4-Three + + + diff --git a/tools/localize/testdata/merge_xx_current.xml b/tools/localize/testdata/merge_xx_current.xml new file mode 100644 index 0000000..c2a783d --- /dev/null +++ b/tools/localize/testdata/merge_xx_current.xml @@ -0,0 +1,22 @@ + + + + + AAAA + CCC + + + diff --git a/tools/localize/testdata/merge_xx_old.xml b/tools/localize/testdata/merge_xx_old.xml new file mode 100644 index 0000000..9d3a7d8 --- /dev/null +++ b/tools/localize/testdata/merge_xx_old.xml @@ -0,0 +1,21 @@ + + + + + aaa + CCC + + diff --git a/tools/localize/testdata/pseudo.xliff b/tools/localize/testdata/pseudo.xliff new file mode 100644 index 0000000..5b44f86 --- /dev/null +++ b/tools/localize/testdata/pseudo.xliff @@ -0,0 +1,40 @@ + + + + + + First underline, italicitalicbold End + + + First underline, italicitalicbold End + + + Simple + + + Simple + + + Simple + + + Quote + + OLD Quote + OLD Ờũỡŧę + + + + + + diff --git a/tools/localize/testdata/res/values-zz-rZZ/strings.xml b/tools/localize/testdata/res/values-zz-rZZ/strings.xml new file mode 100644 index 0000000..c2a783d --- /dev/null +++ b/tools/localize/testdata/res/values-zz-rZZ/strings.xml @@ -0,0 +1,22 @@ + + + + + AAAA + CCC + + + diff --git a/tools/localize/testdata/res/values/strings.xml b/tools/localize/testdata/res/values/strings.xml new file mode 100644 index 0000000..6a11e68 --- /dev/null +++ b/tools/localize/testdata/res/values/strings.xml @@ -0,0 +1,44 @@ + + + + + aaa + bbb + ccc + ccc + + bolditalicitalic_underlineunderline + + + + 1-One + 1-Two + 1-Three + 1-Four + + + + 2-One + 2-Two + 2-Three + + + + 3-One + 3-Two + 3-Three + + diff --git a/tools/localize/testdata/strip_xliff.xliff b/tools/localize/testdata/strip_xliff.xliff new file mode 100644 index 0000000..9254cf2 --- /dev/null +++ b/tools/localize/testdata/strip_xliff.xliff @@ -0,0 +1,70 @@ + + + + + + + + + + source + + + + + target + + + + + source + target + + + + + source + + + source + + source + + + + source + + target + + + + + source + + source + target + + + + source + + alt-source + target + + + + + + + + diff --git a/tools/localize/testdata/values/strings.xml b/tools/localize/testdata/values/strings.xml new file mode 100644 index 0000000..5e8d43d --- /dev/null +++ b/tools/localize/testdata/values/strings.xml @@ -0,0 +1,32 @@ + + + + + Discard + + abcd + abBbC + + + + + Email + Home + Work + Other\u2026 + + diff --git a/tools/localize/testdata/xliff1.xliff b/tools/localize/testdata/xliff1.xliff new file mode 100644 index 0000000..55a8d8e --- /dev/null +++ b/tools/localize/testdata/xliff1.xliff @@ -0,0 +1,46 @@ + + + + + + First underline, italicitalicbold End + Ḟịṙṩŧ , Ḛŋḋ + + + First underline, italicitalicbold End + Ḟịṙṩŧ , Ḛŋḋ + + + Simple + Ṩịṃṕļę + + + Simple + Ṩịṃṕļę + + + Simple + Ṩịṃṕļę + + + Quote + Ờũỡŧę + + OLD Quote + OLD Ờũỡŧę + + + + + + diff --git a/tools/localize/testdata/xml.xml b/tools/localize/testdata/xml.xml new file mode 100644 index 0000000..ef930d0 --- /dev/null +++ b/tools/localize/testdata/xml.xml @@ -0,0 +1,16 @@ + + + + + asdf + + + a,b + a,b + + + diff --git a/tools/localize/xmb.cpp b/tools/localize/xmb.cpp new file mode 100644 index 0000000..236705f --- /dev/null +++ b/tools/localize/xmb.cpp @@ -0,0 +1,181 @@ +#include "xmb.h" + +#include "file_utils.h" +#include "localize.h" +#include "ValuesFile.h" +#include "XMLHandler.h" +#include "XLIFFFile.h" + +#include + +using namespace std; + +const char *const NS_MAP[] = { + "xml", XMLNS_XMLNS, + NULL, NULL +}; + +set g_tags; + +static string +strip_newlines(const string& str) +{ + string res; + const size_t N = str.length(); + for (size_t i=0; i& attrs = node->EditAttributes(); + const size_t I = attrs.size(); + for (size_t i=0; iType() == XMLNode::ELEMENT) { + if (node->Namespace() == XLIFF_XMLNS) { + g_tags.insert(node->Name()); + node->SetName("", "ph"); + + err = rename_id_attribute(node); + if (err != 0) { + char name[30]; + (*phID)++; + sprintf(name, "id-%d", *phID); + node->EditAttributes().push_back(XMLAttribute("", "name", name)); + err = 0; + } + } + vector& children = node->EditChildren(); + const size_t I = children.size(); + for (size_t i=0; i attrs; + string name = res.pos.file; + name += ":"; + name += res.TypedID(); + attrs.push_back(XMLAttribute("", "name", name)); + attrs.push_back(XMLAttribute("", "desc", strip_newlines(res.comment))); + attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve")); + XMLNode* msg = XMLNode::NewElement(res.pos, "", "msg", attrs, XMLNode::EXACT); + + // the contents are in xliff/html, convert it to xliff + int err = 0; + XMLNode* value = res.value; + string tag = value->Name(); + int phID = 0; + for (vector::const_iterator it=value->Children().begin(); + it!=value->Children().end(); it++) { + err |= convert_html_to_xliff(*it, tag, msg, &phID); + } + + if (err != 0) { + return NULL; + } + + // and then convert that to xmb + for (vector::iterator it=msg->EditChildren().begin(); + it!=msg->EditChildren().end(); it++) { + err |= convert_xliff_to_ph(*it, &phID); + } + + if (err == 0) { + return msg; + } else { + return NULL; + } +} + +int +do_xlb_export(const string& outfile, const vector& resFiles) +{ + int err = 0; + + size_t totalFileCount = resFiles.size(); + + Configuration english; + english.locale = "en_US"; + + set allResources; + + const size_t J = resFiles.size(); + for (size_t j=0; j resources = valuesFile->GetStrings(); + allResources.insert(resources.begin(), resources.end()); + } else { + fprintf(stderr, "error reading file %s\n", resFile.c_str()); + } + + delete valuesFile; + } + + // Construct the XLB xml + vector attrs; + attrs.push_back(XMLAttribute("", "locale", "en")); + XMLNode* localizationbundle = XMLNode::NewElement(GENERATED_POS, "", "localizationbundle", + attrs, XMLNode::PRETTY); + + for (set::iterator it=allResources.begin(); it!=allResources.end(); it++) { + XMLNode* msg = resource_to_xmb_msg(*it); + if (msg) { + localizationbundle->EditChildren().push_back(msg); + } else { + err = 1; + } + } + +#if 0 + for (set::iterator it=g_tags.begin(); it!=g_tags.end(); it++) { + printf("tag: %s\n", it->c_str()); + } + printf("err=%d\n", err); +#endif + if (err == 0) { + FILE* f = fopen(outfile.c_str(), "wb"); + if (f == NULL) { + fprintf(stderr, "can't open outputfile: %s\n", outfile.c_str()); + return 1; + } + fprintf(f, "\n"); + fprintf(f, "%s\n", localizationbundle->ToString(NS_MAP).c_str()); + fclose(f); + } + + return err; +} + diff --git a/tools/localize/xmb.h b/tools/localize/xmb.h new file mode 100644 index 0000000..96492b1 --- /dev/null +++ b/tools/localize/xmb.h @@ -0,0 +1,11 @@ +#ifndef XMB_H +#define XMB_H + +#include +#include + +using namespace std; + +int do_xlb_export(const string& outFile, const vector& resFiles); + +#endif // XMB_H -- cgit v1.1