diff options
author | The Android Open Source Project <initial-contribution@android.com> | 2009-03-03 19:31:44 -0800 |
---|---|---|
committer | The Android Open Source Project <initial-contribution@android.com> | 2009-03-03 19:31:44 -0800 |
commit | 9066cfe9886ac131c34d59ed0e2d287b0e3c0087 (patch) | |
tree | d88beb88001f2482911e3d28e43833b50e4b4e97 /tools/localize | |
parent | d83a98f4ce9cfa908f5c54bbd70f03eec07e7553 (diff) | |
download | frameworks_base-9066cfe9886ac131c34d59ed0e2d287b0e3c0087.zip frameworks_base-9066cfe9886ac131c34d59ed0e2d287b0e3c0087.tar.gz frameworks_base-9066cfe9886ac131c34d59ed0e2d287b0e3c0087.tar.bz2 |
auto import from //depot/cupcake/@135843
Diffstat (limited to 'tools/localize')
48 files changed, 5749 insertions, 0 deletions
diff --git a/tools/localize/Android.mk b/tools/localize/Android.mk new file mode 100644 index 0000000..186177f --- /dev/null +++ b/tools/localize/Android.mk @@ -0,0 +1,56 @@ +# +# Copyright 2006 The Android Open Source Project +# +# Android Asset Packaging Tool +# + +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_SRC_FILES := \ + file_utils.cpp \ + localize.cpp \ + merge_res_and_xliff.cpp \ + res_check.cpp \ + xmb.cpp \ + Configuration.cpp \ + Perforce.cpp \ + SourcePos.cpp \ + Values.cpp \ + ValuesFile.cpp \ + XLIFFFile.cpp \ + XMLHandler.cpp + +LOCAL_C_INCLUDES := \ + external/expat/lib \ + build/libs/host/include + +LOCAL_CFLAGS += -g -O0 + +LOCAL_STATIC_LIBRARIES := \ + libexpat \ + libhost \ + libutils \ + libcutils + +ifeq ($(HOST_OS),linux) +LOCAL_LDLIBS += -lrt +endif + + +LOCAL_MODULE := localize + +ifeq (a,a) + LOCAL_CFLAGS += -DLOCALIZE_WITH_TESTS + LOCAL_SRC_FILES += \ + test.cpp \ + localize_test.cpp \ + merge_res_and_xliff_test.cpp \ + Perforce_test.cpp \ + ValuesFile_test.cpp \ + XLIFFFile_test.cpp \ + XMLHandler_test.cpp +endif + +include $(BUILD_HOST_EXECUTABLE) + diff --git a/tools/localize/Configuration.cpp b/tools/localize/Configuration.cpp new file mode 100644 index 0000000..56addbd --- /dev/null +++ b/tools/localize/Configuration.cpp @@ -0,0 +1,76 @@ +#include "Configuration.h" +#include <string.h> + +int +Configuration::Compare(const Configuration& that) const +{ + int n; + + n = locale.compare(that.locale); + if (n != 0) return n; + + n = vendor.compare(that.vendor); + if (n != 0) return n; + + n = orientation.compare(that.orientation); + if (n != 0) return n; + + n = density.compare(that.density); + if (n != 0) return n; + + n = touchscreen.compare(that.touchscreen); + if (n != 0) return n; + + n = keyboard.compare(that.keyboard); + if (n != 0) return n; + + n = navigation.compare(that.navigation); + if (n != 0) return n; + + n = screenSize.compare(that.screenSize); + if (n != 0) return n; + + return 0; +} + +string +Configuration::ToString() const +{ + string s; + if (locale.length() > 0) { + if (s.length() > 0) { + s += "-"; + } + s += locale; + } + return s; +} + +bool +split_locale(const string& in, string* language, string* region) +{ + const int len = in.length(); + if (len == 2) { + if (isalpha(in[0]) && isalpha(in[1])) { + *language = in; + region->clear(); + return true; + } else { + return false; + } + } + else if (len == 5) { + if (isalpha(in[0]) && isalpha(in[1]) && (in[2] == '_' || in[2] == '-') + && isalpha(in[3]) && isalpha(in[4])) { + language->assign(in.c_str(), 2); + region->assign(in.c_str()+3, 2); + return true; + } else { + return false; + } + } + else { + return false; + } +} + diff --git a/tools/localize/Configuration.h b/tools/localize/Configuration.h new file mode 100644 index 0000000..f91bf04 --- /dev/null +++ b/tools/localize/Configuration.h @@ -0,0 +1,38 @@ +#ifndef CONFIGURATION_H +#define CONFIGURATION_H + +#include <string> + +using namespace std; + +struct Configuration +{ + string locale; + string vendor; + string orientation; + string density; + string touchscreen; + string keyboard; + string navigation; + string screenSize; + + // Compare two configurations + int Compare(const Configuration& that) const; + + inline bool operator<(const Configuration& that) const { return Compare(that) < 0; } + inline bool operator<=(const Configuration& that) const { return Compare(that) <= 0; } + inline bool operator==(const Configuration& that) const { return Compare(that) == 0; } + inline bool operator!=(const Configuration& that) const { return Compare(that) != 0; } + inline bool operator>=(const Configuration& that) const { return Compare(that) >= 0; } + inline bool operator>(const Configuration& that) const { return Compare(that) > 0; } + + // Parse a directory name, like "values-en-rUS". Return the first segment in resType. + bool ParseDiectoryName(const string& dir, string* resType); + + string ToString() const; +}; + +bool split_locale(const string& in, string* language, string* region); + + +#endif // CONFIGURATION_H diff --git a/tools/localize/Perforce.cpp b/tools/localize/Perforce.cpp new file mode 100644 index 0000000..3425668 --- /dev/null +++ b/tools/localize/Perforce.cpp @@ -0,0 +1,230 @@ +#include "Perforce.h" +#include "log.h" +#include <string.h> +#include <stdlib.h> +#include <sstream> +#include <sys/types.h> +#include <unistd.h> +#include <sys/wait.h> + +using namespace std; + +extern char** environ; + +int +Perforce::RunCommand(const string& cmd, string* result, bool printOnFailure) +{ + int err; + int outPipe[2]; + int errPipe[2]; + pid_t pid; + + log_printf("Perforce::RunCommand: %s\n", cmd.c_str()); + + err = pipe(outPipe); + err |= pipe(errPipe); + if (err == -1) { + printf("couldn't create pipe. exiting.\n"); + exit(1); + return -1; + } + + pid = fork(); + if (pid == -1) { + printf("couldn't fork. eixiting\n"); + exit(1); + return -1; + } + else if (pid == 0) { + char const* args[] = { + "/bin/sh", + "-c", + cmd.c_str(), + NULL + }; + close(outPipe[0]); + close(errPipe[0]); + dup2(outPipe[1], 1); + dup2(errPipe[1], 2); + execve(args[0], (char* const*)args, environ); + // done + } + + close(outPipe[1]); + close(errPipe[1]); + + result->clear(); + + char buf[1024]; + + // stdout + while (true) { + size_t amt = read(outPipe[0], buf, sizeof(buf)); + result->append(buf, amt); + if (amt <= 0) { + break; + } + } + + // stderr -- the messages are short so it ought to just fit in the buffer + string error; + while (true) { + size_t amt = read(errPipe[0], buf, sizeof(buf)); + error.append(buf, amt); + if (amt <= 0) { + break; + } + } + + close(outPipe[0]); + close(errPipe[0]); + + waitpid(pid, &err, 0); + if (WIFEXITED(err)) { + err = WEXITSTATUS(err); + } else { + err = -1; + } + if (err != 0 && printOnFailure) { + write(2, error.c_str(), error.length()); + } + return err; +} + +int +Perforce::GetResourceFileNames(const string& version, const string& base, + const vector<string>& apps, vector<string>* results, + bool printOnFailure) +{ + int err; + string text; + stringstream cmd; + + cmd << "p4 files"; + + const size_t I = apps.size(); + for (size_t i=0; i<I; i++) { + cmd << " \"" << base << '/' << apps[i] << "/res/values/strings.xml@" << version << '"'; + } + + err = RunCommand(cmd.str(), &text, printOnFailure); + + const char* str = text.c_str(); + while (*str) { + const char* lineend = strchr(str, '\n'); + if (lineend == str) { + str++; + continue; + } + if (lineend-str > 1023) { + fprintf(stderr, "line too long!\n"); + return 1; + } + + string s(str, lineend-str); + + char filename[1024]; + char edit[1024]; + int count = sscanf(str, "%[^#]#%*d - %s change %*d %*[^\n]\n", filename, edit); + + if (count == 2 && 0 != strcmp("delete", edit)) { + results->push_back(string(filename)); + } + + str = lineend + 1; + } + + return err; +} + +int +Perforce::GetFile(const string& file, const string& version, string* result, + bool printOnFailure) +{ + stringstream cmd; + cmd << "p4 print -q \"" << file << '@' << version << '"'; + return RunCommand(cmd.str(), result, printOnFailure); +} + +string +Perforce::GetCurrentChange(bool printOnFailure) +{ + int err; + string text; + + err = RunCommand("p4 changes -m 1 \\#have", &text, printOnFailure); + if (err != 0) { + return ""; + } + + long long n; + int count = sscanf(text.c_str(), "Change %lld on", &n); + if (count != 1) { + return ""; + } + + char result[100]; + sprintf(result, "%lld", n); + + return string(result); +} + +static int +do_files(const string& op, const vector<string>& files, bool printOnFailure) +{ + string text; + stringstream cmd; + + cmd << "p4 " << op; + + const size_t I = files.size(); + for (size_t i=0; i<I; i++) { + cmd << " \"" << files[i] << "\""; + } + + return Perforce::RunCommand(cmd.str(), &text, printOnFailure); +} + +int +Perforce::EditFiles(const vector<string>& files, bool printOnFailure) +{ + return do_files("edit", files, printOnFailure); +} + +int +Perforce::AddFiles(const vector<string>& files, bool printOnFailure) +{ + return do_files("add", files, printOnFailure); +} + +int +Perforce::DeleteFiles(const vector<string>& files, bool printOnFailure) +{ + return do_files("delete", files, printOnFailure); +} + +string +Perforce::Where(const string& depotPath, bool printOnFailure) +{ + int err; + string text; + string cmd = "p4 where "; + cmd += depotPath; + + err = RunCommand(cmd, &text, printOnFailure); + if (err != 0) { + return ""; + } + + size_t index = text.find(' '); + if (index == text.npos) { + return ""; + } + index = text.find(' ', index+1)+1; + if (index == text.npos) { + return ""; + } + + return text.substr(index, text.length()-index-1); +} + diff --git a/tools/localize/Perforce.h b/tools/localize/Perforce.h new file mode 100644 index 0000000..522797d --- /dev/null +++ b/tools/localize/Perforce.h @@ -0,0 +1,25 @@ +#ifndef PERFORCE_H +#define PERFORCE_H + +#include <string> +#include <vector> + +using namespace std; + +class Perforce +{ +public: + static int RunCommand(const string& cmd, string* result, bool printOnFailure); + static int GetResourceFileNames(const string& version, const string& base, + const vector<string>& apps, vector<string>* result, + bool printOnFailure); + static int GetFile(const string& file, const string& version, string* result, + bool printOnFailure); + static string GetCurrentChange(bool printOnFailure); + static int EditFiles(const vector<string>& filename, bool printOnFailure); + static int AddFiles(const vector<string>& files, bool printOnFailure); + static int DeleteFiles(const vector<string>& files, bool printOnFailure); + static string Where(const string& depotPath, bool printOnFailure); +}; + +#endif // PERFORCE_H diff --git a/tools/localize/Perforce_test.cpp b/tools/localize/Perforce_test.cpp new file mode 100644 index 0000000..142b20e --- /dev/null +++ b/tools/localize/Perforce_test.cpp @@ -0,0 +1,62 @@ +#include "Perforce.h" +#include <stdio.h> + +static int +RunCommand_test() +{ + string result; + int err = Perforce::RunCommand("p4 help csommands", &result, true); + printf("err=%d result=[[%s]]\n", err, result.c_str()); + return 0; +} + +static int +GetResourceFileNames_test() +{ + vector<string> results; + vector<string> apps; + apps.push_back("apps/common"); + apps.push_back("apps/Contacts"); + int err = Perforce::GetResourceFileNames("43019", "//device", apps, &results, true); + if (err != 0) { + return err; + } + if (results.size() != 2) { + return 1; + } + if (results[0] != "//device/apps/common/res/values/strings.xml") { + return 1; + } + if (results[1] != "//device/apps/Contacts/res/values/strings.xml") { + return 1; + } + if (false) { + for (size_t i=0; i<results.size(); i++) { + printf("[%zd] '%s'\n", i, results[i].c_str()); + } + } + return 0; +} + +static int +GetFile_test() +{ + string result; + int err = Perforce::GetFile("//device/Makefile", "296", &result, true); + printf("err=%d result=[[%s]]\n", err, result.c_str()); + return 0; +} + +int +Perforce_test() +{ + bool all = false; + int err = 0; + + if (all) err |= RunCommand_test(); + if (all) err |= GetResourceFileNames_test(); + if (all) err |= GetFile_test(); + + return err; +} + diff --git a/tools/localize/SourcePos.cpp b/tools/localize/SourcePos.cpp new file mode 100644 index 0000000..9d7c5c6 --- /dev/null +++ b/tools/localize/SourcePos.cpp @@ -0,0 +1,166 @@ +#include "SourcePos.h" + +#include <stdarg.h> +#include <set> + +using namespace std; + +const SourcePos GENERATED_POS("<generated>", -1); + +// ErrorPos +// ============================================================================= +struct ErrorPos +{ + string file; + int line; + string error; + + ErrorPos(); + ErrorPos(const ErrorPos& that); + ErrorPos(const string& file, int line, const string& error); + ~ErrorPos(); + bool operator<(const ErrorPos& rhs) const; + bool operator==(const ErrorPos& rhs) const; + ErrorPos& operator=(const ErrorPos& rhs); + + void Print(FILE* to) const; +}; + +static set<ErrorPos> g_errors; + +ErrorPos::ErrorPos() +{ +} + +ErrorPos::ErrorPos(const ErrorPos& that) + :file(that.file), + line(that.line), + error(that.error) +{ +} + +ErrorPos::ErrorPos(const string& f, int l, const string& e) + :file(f), + line(l), + error(e) +{ +} + +ErrorPos::~ErrorPos() +{ +} + +bool +ErrorPos::operator<(const ErrorPos& rhs) const +{ + if (this->file < rhs.file) return true; + if (this->file == rhs.file) { + if (this->line < rhs.line) return true; + if (this->line == rhs.line) { + if (this->error < rhs.error) return true; + } + } + return false; +} + +bool +ErrorPos::operator==(const ErrorPos& rhs) const +{ + return this->file == rhs.file + && this->line == rhs.line + && this->error == rhs.error; +} + +ErrorPos& +ErrorPos::operator=(const ErrorPos& rhs) +{ + this->file = rhs.file; + this->line = rhs.line; + this->error = rhs.error; + return *this; +} + +void +ErrorPos::Print(FILE* to) const +{ + if (this->line >= 0) { + fprintf(to, "%s:%d: %s\n", this->file.c_str(), this->line, this->error.c_str()); + } else { + fprintf(to, "%s: %s\n", this->file.c_str(), this->error.c_str()); + } +} + +// SourcePos +// ============================================================================= +SourcePos::SourcePos(const string& f, int l) + : file(f), line(l) +{ +} + +SourcePos::SourcePos(const SourcePos& that) + : file(that.file), line(that.line) +{ +} + +SourcePos::SourcePos() + : file("???", 0) +{ +} + +SourcePos::~SourcePos() +{ +} + +string +SourcePos::ToString() const +{ + char buf[1024]; + if (this->line >= 0) { + snprintf(buf, sizeof(buf)-1, "%s:%d", this->file.c_str(), this->line); + } else { + snprintf(buf, sizeof(buf)-1, "%s:", this->file.c_str()); + } + buf[sizeof(buf)-1] = '\0'; + return string(buf); +} + +int +SourcePos::Error(const char* fmt, ...) const +{ + int retval=0; + char buf[1024]; + va_list ap; + va_start(ap, fmt); + retval = vsnprintf(buf, sizeof(buf), fmt, ap); + va_end(ap); + char* p = buf + retval - 1; + while (p > buf && *p == '\n') { + *p = '\0'; + p--; + } + ErrorPos err(this->file, this->line, string(buf)); + if (g_errors.find(err) == g_errors.end()) { + err.Print(stderr); + g_errors.insert(err); + } + return retval; +} + +bool +SourcePos::HasErrors() +{ + return g_errors.size() > 0; +} + +void +SourcePos::PrintErrors(FILE* to) +{ + set<ErrorPos>::const_iterator it; + for (it=g_errors.begin(); it!=g_errors.end(); it++) { + it->Print(to); + } +} + + + + diff --git a/tools/localize/SourcePos.h b/tools/localize/SourcePos.h new file mode 100644 index 0000000..5027129 --- /dev/null +++ b/tools/localize/SourcePos.h @@ -0,0 +1,28 @@ +#ifndef SOURCEPOS_H +#define SOURCEPOS_H + +#include <string> + +using namespace std; + +class SourcePos +{ +public: + string file; + int line; + + SourcePos(const string& f, int l); + SourcePos(const SourcePos& that); + SourcePos(); + ~SourcePos(); + + string ToString() const; + int Error(const char* fmt, ...) const; + + static bool HasErrors(); + static void PrintErrors(FILE* to); +}; + +extern const SourcePos GENERATED_POS; + +#endif // SOURCEPOS_H diff --git a/tools/localize/Values.cpp b/tools/localize/Values.cpp new file mode 100644 index 0000000..e396f8b --- /dev/null +++ b/tools/localize/Values.cpp @@ -0,0 +1,134 @@ +#include "Values.h" +#include <stdlib.h> + + +// ===================================================================================== +StringResource::StringResource(const SourcePos& p, const string& f, const Configuration& c, + const string& i, int ix, XMLNode* v, const int ve, const string& vs, + const string& cmnt) + :pos(p), + file(f), + config(c), + id(i), + index(ix), + value(v), + version(ve), + versionString(vs), + comment(cmnt) +{ +} + +StringResource::StringResource() + :pos(), + file(), + config(), + id(), + index(-1), + value(NULL), + version(), + versionString(), + comment() +{ +} + +StringResource::StringResource(const StringResource& that) + :pos(that.pos), + file(that.file), + config(that.config), + id(that.id), + index(that.index), + value(that.value), + version(that.version), + versionString(that.versionString), + comment(that.comment) +{ +} + +int +StringResource::Compare(const StringResource& that) const +{ + if (file != that.file) { + return file < that.file ? -1 : 1; + } + if (id != that.id) { + return id < that.id ? -1 : 1; + } + if (index != that.index) { + return index - that.index; + } + if (config != that.config) { + return config < that.config ? -1 : 1; + } + if (version != that.version) { + return version < that.version ? -1 : 1; + } + return 0; +} + +string +StringResource::TypedID() const +{ + string result; + if (index < 0) { + result = "string:"; + } else { + char n[20]; + sprintf(n, "%d:", index); + result = "array:"; + result += n; + } + result += id; + return result; +} + +static void +split(const string& raw, vector<string>*parts) +{ + size_t index = 0; + while (true) { + size_t next = raw.find(':', index); + if (next != raw.npos) { + parts->push_back(string(raw, index, next-index)); + index = next + 1; + } else { + parts->push_back(string(raw, index)); + break; + } + } +} + +bool +StringResource::ParseTypedID(const string& raw, string* id, int* index) +{ + vector<string> parts; + split(raw, &parts); + + const size_t N = parts.size(); + + for (size_t i=0; i<N; i++) { + if (parts[i].length() == 0) { + return false; + } + } + + if (N == 2 && parts[0] == "string") { + *id = parts[1]; + *index = -1; + return true; + } + else if (N == 3 && parts[0] == "array") { + char* p; + int n = (int)strtol(parts[1].c_str(), &p, 0); + if (*p == '\0') { + *id = parts[2]; + *index = n; + return true; + } else { + return false; + } + } + else { + return false; + } +} + diff --git a/tools/localize/Values.h b/tools/localize/Values.h new file mode 100644 index 0000000..0a60b6d --- /dev/null +++ b/tools/localize/Values.h @@ -0,0 +1,48 @@ +#ifndef VALUES_H +#define VALUES_H + +#include "Configuration.h" +#include "XMLHandler.h" + +#include <string> + +using namespace std; + +enum { + CURRENT_VERSION, + OLD_VERSION +}; + +struct StringResource +{ + StringResource(); + StringResource(const SourcePos& pos, const string& file, const Configuration& config, + const string& id, int index, XMLNode* value, + int version, const string& versionString, const string& comment = ""); + StringResource(const StringResource& that); + + // Compare two configurations + int Compare(const StringResource& that) const; + + inline bool operator<(const StringResource& that) const { return Compare(that) < 0; } + inline bool operator<=(const StringResource& that) const { return Compare(that) <= 0; } + inline bool operator==(const StringResource& that) const { return Compare(that) == 0; } + inline bool operator!=(const StringResource& that) const { return Compare(that) != 0; } + inline bool operator>=(const StringResource& that) const { return Compare(that) >= 0; } + inline bool operator>(const StringResource& that) const { return Compare(that) > 0; } + + string TypedID() const; + static bool ParseTypedID(const string& typed, string* id, int* index); + + SourcePos pos; + string file; + Configuration config; + string id; + int index; + XMLNode* value; + int version; + string versionString; + string comment; +}; + +#endif // VALUES_H diff --git a/tools/localize/ValuesFile.cpp b/tools/localize/ValuesFile.cpp new file mode 100644 index 0000000..bd6f494 --- /dev/null +++ b/tools/localize/ValuesFile.cpp @@ -0,0 +1,266 @@ +#include "ValuesFile.h" + +#include "XMLHandler.h" + +#include <algorithm> +#include <fcntl.h> +#include <expat.h> +#include <unistd.h> +#include <errno.h> + +using namespace std; + +const char* const ANDROID_XMLNS = "http://schemas.android.com/apk/res/android"; +const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2"; + +const char *const NS_MAP[] = { + "android", ANDROID_XMLNS, + "xliff", XLIFF_XMLNS, + NULL, NULL +}; + +const XMLNamespaceMap ANDROID_NAMESPACES(NS_MAP); + + +// ===================================================================================== +class ArrayHandler : public XMLHandler +{ +public: + ArrayHandler(ValuesFile* vf, int version, const string& versionString, const string& id); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + +private: + ValuesFile* m_vf; + int m_version; + int m_index; + string m_versionString; + string m_id; + string m_comment; +}; + +ArrayHandler::ArrayHandler(ValuesFile* vf, int version, const string& versionString, + const string& id) + :m_vf(vf), + m_version(version), + m_index(0), + m_versionString(versionString), + m_id(id) +{ +} + +int +ArrayHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next) +{ + if (ns == "" && name == "item") { + XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT); + m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(), + m_id, m_index, node, m_version, m_versionString, + trim_string(m_comment))); + *next = new NodeHandler(node, XMLNode::EXACT); + m_index++; + m_comment = ""; + return 0; + } else { + pos.Error("invalid <%s> element inside <array>\n", name.c_str()); + return 1; + } +} + +int +ArrayHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +ArrayHandler::OnComment(const SourcePos& pos, const string& text) +{ + m_comment += text; + return 0; +} + +// ===================================================================================== +class ValuesHandler : public XMLHandler +{ +public: + ValuesHandler(ValuesFile* vf, int version, const string& versionString); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + +private: + ValuesFile* m_vf; + int m_version; + string m_versionString; + string m_comment; +}; + +ValuesHandler::ValuesHandler(ValuesFile* vf, int version, const string& versionString) + :m_vf(vf), + m_version(version), + m_versionString(versionString) +{ +} + +int +ValuesHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next) +{ + if (ns == "" && name == "string") { + string id = XMLAttribute::Find(attrs, "", "name", ""); + XMLNode* node = XMLNode::NewElement(pos, ns, name, attrs, XMLNode::EXACT); + m_vf->AddString(StringResource(pos, pos.file, m_vf->GetConfiguration(), + id, -1, node, m_version, m_versionString, + trim_string(m_comment))); + *next = new NodeHandler(node, XMLNode::EXACT); + } + else if (ns == "" && name == "array") { + string id = XMLAttribute::Find(attrs, "", "name", ""); + *next = new ArrayHandler(m_vf, m_version, m_versionString, id); + } + m_comment = ""; + return 0; +} + +int +ValuesHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +ValuesHandler::OnComment(const SourcePos& pos, const string& text) +{ + m_comment += text; + return 0; +} + +// ===================================================================================== +ValuesFile::ValuesFile(const Configuration& config) + :m_config(config), + m_strings(), + m_arrays() +{ +} + +ValuesFile::~ValuesFile() +{ +} + +ValuesFile* +ValuesFile::ParseFile(const string& filename, const Configuration& config, + int version, const string& versionString) +{ + ValuesFile* result = new ValuesFile(config); + + TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString)); + XMLHandler::ParseFile(filename, &top); + + return result; +} + +ValuesFile* +ValuesFile::ParseString(const string& filename, const string& text, const Configuration& config, + int version, const string& versionString) +{ + ValuesFile* result = new ValuesFile(config); + + TopElementHandler top("", "resources", new ValuesHandler(result, version, versionString)); + XMLHandler::ParseString(filename, text, &top); + + return result; +} + +const Configuration& +ValuesFile::GetConfiguration() const +{ + return m_config; +} + +void +ValuesFile::AddString(const StringResource& str) +{ + if (str.index < 0) { + m_strings.insert(str); + } else { + m_arrays[str.id].insert(str); + } +} + +set<StringResource> +ValuesFile::GetStrings() const +{ + set<StringResource> result = m_strings; + + for (map<string,set<StringResource> >::const_iterator it = m_arrays.begin(); + it != m_arrays.end(); it++) { + result.insert(it->second.begin(), it->second.end()); + } + + return result; +} + +XMLNode* +ValuesFile::ToXMLNode() const +{ + XMLNode* root; + + // <resources> + { + vector<XMLAttribute> attrs; + ANDROID_NAMESPACES.AddToAttributes(&attrs); + root = XMLNode::NewElement(GENERATED_POS, "", "resources", attrs, XMLNode::PRETTY); + } + + // <array> + for (map<string,set<StringResource> >::const_iterator it = m_arrays.begin(); + it != m_arrays.end(); it++) { + vector<XMLAttribute> arrayAttrs; + arrayAttrs.push_back(XMLAttribute("", "name", it->first)); + const set<StringResource>& items = it->second; + XMLNode* arrayNode = XMLNode::NewElement(items.begin()->pos, "", "array", arrayAttrs, + XMLNode::PRETTY); + root->EditChildren().push_back(arrayNode); + + // <item> + for (set<StringResource>::const_iterator item = items.begin(); + item != items.end(); item++) { + XMLNode* itemNode = item->value->Clone(); + itemNode->SetName("", "item"); + itemNode->EditAttributes().clear(); + arrayNode->EditChildren().push_back(itemNode); + } + } + + // <string> + for (set<StringResource>::const_iterator it=m_strings.begin(); it!=m_strings.end(); it++) { + const StringResource& str = *it; + vector<XMLAttribute> attrs; + XMLNode* strNode = str.value->Clone(); + strNode->SetName("", "string"); + strNode->EditAttributes().clear(); + strNode->EditAttributes().push_back(XMLAttribute("", "name", str.id)); + root->EditChildren().push_back(strNode); + } + + return root; +} + +string +ValuesFile::ToString() const +{ + XMLNode* xml = ToXMLNode(); + string s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"; + s += xml->ToString(ANDROID_NAMESPACES); + delete xml; + s += '\n'; + return s; +} + diff --git a/tools/localize/ValuesFile.h b/tools/localize/ValuesFile.h new file mode 100644 index 0000000..752fd78 --- /dev/null +++ b/tools/localize/ValuesFile.h @@ -0,0 +1,52 @@ +#ifndef VALUES_FILE_H +#define VALUES_FILE_H + +#include "SourcePos.h" +#include "Configuration.h" +#include "XMLHandler.h" +#include "Values.h" + +#include <string> +#include <set> + +using namespace std; + +extern const XMLNamespaceMap ANDROID_NAMESPACES; + +class ValuesFile +{ +public: + ValuesFile(const Configuration& config); + + static ValuesFile* ParseFile(const string& filename, const Configuration& config, + int version, const string& versionString); + static ValuesFile* ParseString(const string& filename, const string& text, + const Configuration& config, + int version, const string& versionString); + ~ValuesFile(); + + const Configuration& GetConfiguration() const; + + void AddString(const StringResource& str); + set<StringResource> GetStrings() const; + + // exports this file as a n XMLNode, you own this object + XMLNode* ToXMLNode() const; + + // writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of + // ToXMLNode()). + string ToString() const; + +private: + class ParseState; + friend class ValuesFile::ParseState; + friend class StringHandler; + + ValuesFile(); + + Configuration m_config; + set<StringResource> m_strings; + map<string,set<StringResource> > m_arrays; +}; + +#endif // VALUES_FILE_H diff --git a/tools/localize/ValuesFile_test.cpp b/tools/localize/ValuesFile_test.cpp new file mode 100644 index 0000000..56d2ec2 --- /dev/null +++ b/tools/localize/ValuesFile_test.cpp @@ -0,0 +1,54 @@ +#include "ValuesFile.h" +#include <stdio.h> + +int +ValuesFile_test() +{ + int err = 0; + Configuration config; + config.locale = "zz_ZZ"; + ValuesFile* vf = ValuesFile::ParseFile("testdata/values/strings.xml", config, + OLD_VERSION, "1"); + + const set<StringResource>& strings = vf->GetStrings(); + string canonical = vf->ToString(); + + if (false) { + printf("Strings (%zd)\n", strings.size()); + for (set<StringResource>::const_iterator it=strings.begin(); + it!=strings.end(); it++) { + const StringResource& str = *it; + printf("%s: '%s'[%d]='%s' (%s) <!-- %s -->\n", str.pos.ToString().c_str(), + str.id.c_str(), str.index, + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.config.ToString().c_str(), str.comment.c_str()); + } + + printf("XML:[[%s]]\n", canonical.c_str()); + } + + const char * const EXPECTED = + "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" + "<resources xmlns:android=\"http://schemas.android.com/apk/res/android\"\n" + " xmlns:xliff=\"urn:oasis:names:tc:xliff:document:1.2\">\n" + " <array name=\"emailAddressTypes\">\n" + " <item>Email</item>\n" + " <item>Home</item>\n" + " <item>Work</item>\n" + " <item>Other\\u2026</item>\n" + " </array>\n" + " <string name=\"test1\">Discard</string>\n" + " <string name=\"test2\">a<b>b<i>c</i></b>d</string>\n" + " <string name=\"test3\">a<xliff:g a=\"b\" xliff:a=\"asdf\">bBb</xliff:g>C</string>\n" + "</resources>\n"; + + if (canonical != EXPECTED) { + fprintf(stderr, "ValuesFile_test failed\n"); + fprintf(stderr, "canonical=[[%s]]\n", canonical.c_str()); + fprintf(stderr, "EXPECTED=[[%s]]\n", EXPECTED); + err = 1; + } + + delete vf; + return err; +} diff --git a/tools/localize/XLIFFFile.cpp b/tools/localize/XLIFFFile.cpp new file mode 100644 index 0000000..51f81de --- /dev/null +++ b/tools/localize/XLIFFFile.cpp @@ -0,0 +1,609 @@ +#include "XLIFFFile.h" + +#include <algorithm> +#include <sys/time.h> +#include <time.h> + +const char* const XLIFF_XMLNS = "urn:oasis:names:tc:xliff:document:1.2"; + +const char *const NS_MAP[] = { + "", XLIFF_XMLNS, + "xml", XMLNS_XMLNS, + NULL, NULL +}; + +const XMLNamespaceMap XLIFF_NAMESPACES(NS_MAP); + +int +XLIFFFile::File::Compare(const XLIFFFile::File& that) const +{ + if (filename != that.filename) { + return filename < that.filename ? -1 : 1; + } + return 0; +} + +// ===================================================================================== +XLIFFFile::XLIFFFile() +{ +} + +XLIFFFile::~XLIFFFile() +{ +} + +static XMLNode* +get_unique_node(const XMLNode* parent, const string& ns, const string& name, bool required) +{ + size_t count = parent->CountElementsByName(ns, name); + if (count == 1) { + return parent->GetElementByNameAt(ns, name, 0); + } else { + if (required) { + SourcePos pos = count == 0 + ? parent->Position() + : parent->GetElementByNameAt(XLIFF_XMLNS, name, 1)->Position(); + pos.Error("<%s> elements must contain exactly one <%s> element", + parent->Name().c_str(), name.c_str()); + } + return NULL; + } +} + +XLIFFFile* +XLIFFFile::Parse(const string& filename) +{ + XLIFFFile* result = new XLIFFFile(); + + XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY); + if (root == NULL) { + return NULL; + } + + // <file> + vector<XMLNode*> files = root->GetElementsByName(XLIFF_XMLNS, "file"); + for (size_t i=0; i<files.size(); i++) { + XMLNode* file = files[i]; + + string datatype = file->GetAttribute("", "datatype", ""); + string originalFile = file->GetAttribute("", "original", ""); + + Configuration sourceConfig; + sourceConfig.locale = file->GetAttribute("", "source-language", ""); + result->m_sourceConfig = sourceConfig; + + Configuration targetConfig; + targetConfig.locale = file->GetAttribute("", "target-language", ""); + result->m_targetConfig = targetConfig; + + result->m_currentVersion = file->GetAttribute("", "build-num", ""); + result->m_oldVersion = "old"; + + // <body> + XMLNode* body = get_unique_node(file, XLIFF_XMLNS, "body", true); + if (body == NULL) continue; + + // <trans-unit> + vector<XMLNode*> transUnits = body->GetElementsByName(XLIFF_XMLNS, "trans-unit"); + for (size_t j=0; j<transUnits.size(); j++) { + XMLNode* transUnit = transUnits[j]; + + string rawID = transUnit->GetAttribute("", "id", ""); + if (rawID == "") { + transUnit->Position().Error("<trans-unit> tag requires an id"); + continue; + } + string id; + int index; + + if (!StringResource::ParseTypedID(rawID, &id, &index)) { + transUnit->Position().Error("<trans-unit> has invalid id '%s'\n", rawID.c_str()); + continue; + } + + // <source> + XMLNode* source = get_unique_node(transUnit, XLIFF_XMLNS, "source", false); + if (source != NULL) { + XMLNode* node = source->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(source->Position(), originalFile, + sourceConfig, id, index, node, CURRENT_VERSION, + result->m_currentVersion)); + } + + // <target> + XMLNode* target = get_unique_node(transUnit, XLIFF_XMLNS, "target", false); + if (target != NULL) { + XMLNode* node = target->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(target->Position(), originalFile, + targetConfig, id, index, node, CURRENT_VERSION, + result->m_currentVersion)); + } + + // <alt-trans> + XMLNode* altTrans = get_unique_node(transUnit, XLIFF_XMLNS, "alt-trans", false); + if (altTrans != NULL) { + // <source> + XMLNode* altSource = get_unique_node(altTrans, XLIFF_XMLNS, "source", false); + if (altSource != NULL) { + XMLNode* node = altSource->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(altSource->Position(), + originalFile, sourceConfig, id, index, node, OLD_VERSION, + result->m_oldVersion)); + } + + // <target> + XMLNode* altTarget = get_unique_node(altTrans, XLIFF_XMLNS, "target", false); + if (altTarget != NULL) { + XMLNode* node = altTarget->Clone(); + node->SetPrettyRecursive(XMLNode::EXACT); + result->AddStringResource(StringResource(altTarget->Position(), + originalFile, targetConfig, id, index, node, OLD_VERSION, + result->m_oldVersion)); + } + } + } + } + delete root; + return result; +} + +XLIFFFile* +XLIFFFile::Create(const Configuration& sourceConfig, const Configuration& targetConfig, + const string& currentVersion) +{ + XLIFFFile* result = new XLIFFFile(); + result->m_sourceConfig = sourceConfig; + result->m_targetConfig = targetConfig; + result->m_currentVersion = currentVersion; + return result; +} + +set<string> +XLIFFFile::Files() const +{ + set<string> result; + for (vector<File>::const_iterator f = m_files.begin(); f != m_files.end(); f++) { + result.insert(f->filename); + } + return result; +} + +void +XLIFFFile::AddStringResource(const StringResource& str) +{ + string id = str.TypedID(); + + File* f = NULL; + const size_t I = m_files.size(); + for (size_t i=0; i<I; i++) { + if (m_files[i].filename == str.file) { + f = &m_files[i]; + break; + } + } + if (f == NULL) { + File file; + file.filename = str.file; + m_files.push_back(file); + f = &m_files[I]; + } + + const size_t J = f->transUnits.size(); + TransUnit* g = NULL; + for (size_t j=0; j<J; j++) { + if (f->transUnits[j].id == id) { + g = &f->transUnits[j]; + } + } + if (g == NULL) { + TransUnit group; + group.id = id; + f->transUnits.push_back(group); + g = &f->transUnits[J]; + } + + StringResource* res = find_string_res(*g, str); + if (res == NULL) { + return ; + } + if (res->id != "") { + str.pos.Error("Duplicate string resource: %s", res->id.c_str()); + res->pos.Error("Previous definition here"); + return ; + } + *res = str; + + m_strings.insert(str); +} + +void +XLIFFFile::Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie) +{ + const size_t I = m_files.size(); + for (size_t ix=0, i=I-1; ix<I; ix++, i--) { + File& file = m_files[i]; + + const size_t J = file.transUnits.size(); + for (size_t jx=0, j=J-1; jx<J; jx++, j--) { + TransUnit& tu = file.transUnits[j]; + + bool keep = func(file.filename, tu, cookie); + if (!keep) { + if (tu.source.id != "") { + m_strings.erase(tu.source); + } + if (tu.target.id != "") { + m_strings.erase(tu.target); + } + if (tu.altSource.id != "") { + m_strings.erase(tu.altSource); + } + if (tu.altTarget.id != "") { + m_strings.erase(tu.altTarget); + } + file.transUnits.erase(file.transUnits.begin()+j); + } + } + if (file.transUnits.size() == 0) { + m_files.erase(m_files.begin()+i); + } + } +} + +void +XLIFFFile::Map(void (*func)(const string&,TransUnit*,void*), void* cookie) +{ + const size_t I = m_files.size(); + for (size_t i=0; i<I; i++) { + File& file = m_files[i]; + + const size_t J = file.transUnits.size(); + for (size_t j=0; j<J; j++) { + func(file.filename, &(file.transUnits[j]), cookie); + } + } +} + +TransUnit* +XLIFFFile::EditTransUnit(const string& filename, const string& id) +{ + const size_t I = m_files.size(); + for (size_t ix=0, i=I-1; ix<I; ix++, i--) { + File& file = m_files[i]; + if (file.filename == filename) { + const size_t J = file.transUnits.size(); + for (size_t jx=0, j=J-1; jx<J; jx++, j--) { + TransUnit& tu = file.transUnits[j]; + if (tu.id == id) { + return &tu; + } + } + } + } + return NULL; +} + +StringResource* +XLIFFFile::find_string_res(TransUnit& g, const StringResource& str) +{ + int index; + if (str.version == CURRENT_VERSION) { + index = 0; + } + else if (str.version == OLD_VERSION) { + index = 2; + } + else { + str.pos.Error("Internal Error %s:%d\n", __FILE__, __LINE__); + return NULL; + } + if (str.config == m_sourceConfig) { + // index += 0; + } + else if (str.config == m_targetConfig) { + index += 1; + } + else { + str.pos.Error("unknown config for string %s: %s", str.id.c_str(), + str.config.ToString().c_str()); + return NULL; + } + switch (index) { + case 0: + return &g.source; + case 1: + return &g.target; + case 2: + return &g.altSource; + case 3: + return &g.altTarget; + } + str.pos.Error("Internal Error %s:%d\n", __FILE__, __LINE__); + return NULL; +} + +int +convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID) +{ + int err = 0; + if (original->Type() == XMLNode::TEXT) { + addTo->EditChildren().push_back(original->Clone()); + return 0; + } else { + string ctype; + if (original->Namespace() == "") { + if (original->Name() == "b") { + ctype = "bold"; + } + else if (original->Name() == "i") { + ctype = "italic"; + } + else if (original->Name() == "u") { + ctype = "underline"; + } + } + if (ctype != "") { + vector<XMLAttribute> attrs; + attrs.push_back(XMLAttribute(XLIFF_XMLNS, "ctype", ctype)); + XMLNode* copy = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, "g", + attrs, XMLNode::EXACT); + + const vector<XMLNode*>& children = original->Children(); + size_t I = children.size(); + for (size_t i=0; i<I; i++) { + err |= convert_html_to_xliff(children[i], name, copy, phID); + } + return err; + } + else { + if (original->Namespace() == XLIFF_XMLNS) { + addTo->EditChildren().push_back(original->Clone()); + return 0; + } else { + if (original->Namespace() == "") { + // flatten out the tag into ph tags -- but only if there is no namespace + // that's still unsupported because propagating the xmlns attribute is hard. + vector<XMLAttribute> attrs; + char idStr[30]; + (*phID)++; + sprintf(idStr, "id-%d", *phID); + attrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", idStr)); + + if (original->Children().size() == 0) { + XMLNode* ph = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, + "ph", attrs, XMLNode::EXACT); + ph->EditChildren().push_back( + XMLNode::NewText(original->Position(), + original->ToString(XLIFF_NAMESPACES), + XMLNode::EXACT)); + addTo->EditChildren().push_back(ph); + } else { + XMLNode* begin = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, + "bpt", attrs, XMLNode::EXACT); + begin->EditChildren().push_back( + XMLNode::NewText(original->Position(), + original->OpenTagToString(XLIFF_NAMESPACES, XMLNode::EXACT), + XMLNode::EXACT)); + XMLNode* end = XMLNode::NewElement(original->Position(), XLIFF_XMLNS, + "ept", attrs, XMLNode::EXACT); + string endText = "</"; + endText += original->Name(); + endText += ">"; + end->EditChildren().push_back(XMLNode::NewText(original->Position(), + endText, XMLNode::EXACT)); + + addTo->EditChildren().push_back(begin); + + const vector<XMLNode*>& children = original->Children(); + size_t I = children.size(); + for (size_t i=0; i<I; i++) { + err |= convert_html_to_xliff(children[i], name, addTo, phID); + } + + addTo->EditChildren().push_back(end); + } + return err; + } else { + original->Position().Error("invalid <%s> element in <%s> tag\n", + original->Name().c_str(), name.c_str()); + return 1; + } + } + } + } +} + +XMLNode* +create_string_node(const StringResource& str, const string& name) +{ + vector<XMLAttribute> attrs; + attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve")); + XMLNode* node = XMLNode::NewElement(str.pos, XLIFF_XMLNS, name, attrs, XMLNode::EXACT); + + const vector<XMLNode*>& children = str.value->Children(); + size_t I = children.size(); + int err = 0; + for (size_t i=0; i<I; i++) { + int phID = 0; + err |= convert_html_to_xliff(children[i], name, node, &phID); + } + + if (err != 0) { + delete node; + } + return node; +} + +static bool +compare_id(const TransUnit& lhs, const TransUnit& rhs) +{ + string lid, rid; + int lindex, rindex; + StringResource::ParseTypedID(lhs.id, &lid, &lindex); + StringResource::ParseTypedID(rhs.id, &rid, &rindex); + if (lid < rid) return true; + if (lid == rid && lindex < rindex) return true; + return false; +} + +XMLNode* +XLIFFFile::ToXMLNode() const +{ + XMLNode* root; + size_t N; + + // <xliff> + { + vector<XMLAttribute> attrs; + XLIFF_NAMESPACES.AddToAttributes(&attrs); + attrs.push_back(XMLAttribute(XLIFF_XMLNS, "version", "1.2")); + root = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "xliff", attrs, XMLNode::PRETTY); + } + + vector<TransUnit> groups; + + // <file> + vector<File> files = m_files; + sort(files.begin(), files.end()); + const size_t I = files.size(); + for (size_t i=0; i<I; i++) { + const File& file = files[i]; + + vector<XMLAttribute> fileAttrs; + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "datatype", "x-android-res")); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "original", file.filename)); + + struct timeval tv; + struct timezone tz; + gettimeofday(&tv, &tz); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "date", trim_string(ctime(&tv.tv_sec)))); + + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "source-language", m_sourceConfig.locale)); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "target-language", m_targetConfig.locale)); + fileAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "build-num", m_currentVersion)); + + XMLNode* fileNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "file", fileAttrs, + XMLNode::PRETTY); + root->EditChildren().push_back(fileNode); + + // <body> + XMLNode* bodyNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "body", + vector<XMLAttribute>(), XMLNode::PRETTY); + fileNode->EditChildren().push_back(bodyNode); + + // <trans-unit> + vector<TransUnit> transUnits = file.transUnits; + sort(transUnits.begin(), transUnits.end(), compare_id); + const size_t J = transUnits.size(); + for (size_t j=0; j<J; j++) { + const TransUnit& transUnit = transUnits[j]; + + vector<XMLAttribute> tuAttrs; + + // strings start with string: + tuAttrs.push_back(XMLAttribute(XLIFF_XMLNS, "id", transUnit.id)); + XMLNode* transUnitNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "trans-unit", + tuAttrs, XMLNode::PRETTY); + bodyNode->EditChildren().push_back(transUnitNode); + + // <extradata> + if (transUnit.source.comment != "") { + vector<XMLAttribute> extradataAttrs; + XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "extradata", + extradataAttrs, XMLNode::EXACT); + transUnitNode->EditChildren().push_back(extraNode); + extraNode->EditChildren().push_back( + XMLNode::NewText(GENERATED_POS, transUnit.source.comment, + XMLNode::PRETTY)); + } + + // <source> + if (transUnit.source.id != "") { + transUnitNode->EditChildren().push_back( + create_string_node(transUnit.source, "source")); + } + + // <target> + if (transUnit.target.id != "") { + transUnitNode->EditChildren().push_back( + create_string_node(transUnit.target, "target")); + } + + // <alt-trans> + if (transUnit.altSource.id != "" || transUnit.altTarget.id != "" + || transUnit.rejectComment != "") { + vector<XMLAttribute> altTransAttrs; + XMLNode* altTransNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, "alt-trans", + altTransAttrs, XMLNode::PRETTY); + transUnitNode->EditChildren().push_back(altTransNode); + + // <extradata> + if (transUnit.rejectComment != "") { + vector<XMLAttribute> extradataAttrs; + XMLNode* extraNode = XMLNode::NewElement(GENERATED_POS, XLIFF_XMLNS, + "extradata", extradataAttrs, + XMLNode::EXACT); + altTransNode->EditChildren().push_back(extraNode); + extraNode->EditChildren().push_back( + XMLNode::NewText(GENERATED_POS, transUnit.rejectComment, + XMLNode::PRETTY)); + } + + // <source> + if (transUnit.altSource.id != "") { + altTransNode->EditChildren().push_back( + create_string_node(transUnit.altSource, "source")); + } + + // <target> + if (transUnit.altTarget.id != "") { + altTransNode->EditChildren().push_back( + create_string_node(transUnit.altTarget, "target")); + } + } + + } + } + + return root; +} + + +string +XLIFFFile::ToString() const +{ + XMLNode* xml = ToXMLNode(); + string s = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"; + s += xml->ToString(XLIFF_NAMESPACES); + delete xml; + s += '\n'; + return s; +} + +Stats +XLIFFFile::GetStats(const string& config) const +{ + Stats stat; + stat.config = config; + stat.files = m_files.size(); + stat.toBeTranslated = 0; + stat.noComments = 0; + + for (vector<File>::const_iterator file=m_files.begin(); file!=m_files.end(); file++) { + stat.toBeTranslated += file->transUnits.size(); + + for (vector<TransUnit>::const_iterator tu=file->transUnits.begin(); + tu!=file->transUnits.end(); tu++) { + if (tu->source.comment == "") { + stat.noComments++; + } + } + } + + stat.totalStrings = stat.toBeTranslated; + + return stat; +} diff --git a/tools/localize/XLIFFFile.h b/tools/localize/XLIFFFile.h new file mode 100644 index 0000000..a93d479 --- /dev/null +++ b/tools/localize/XLIFFFile.h @@ -0,0 +1,98 @@ +#ifndef XLIFF_FILE_H +#define XLIFF_FILE_H + +#include "Values.h" + +#include "Configuration.h" + +#include <set> + +using namespace std; + +extern const XMLNamespaceMap XLIFF_NAMESPACES; + +extern const char*const XLIFF_XMLNS; + +struct Stats +{ + string config; + size_t files; + size_t toBeTranslated; + size_t noComments; + size_t totalStrings; +}; + +struct TransUnit { + string id; + StringResource source; + StringResource target; + StringResource altSource; + StringResource altTarget; + string rejectComment; +}; + +class XLIFFFile +{ +public: + static XLIFFFile* Parse(const string& filename); + static XLIFFFile* Create(const Configuration& sourceConfig, const Configuration& targetConfig, + const string& currentVersion); + ~XLIFFFile(); + + inline const Configuration& SourceConfig() const { return m_sourceConfig; } + inline const Configuration& TargetConfig() const { return m_targetConfig; } + + inline const string& CurrentVersion() const { return m_currentVersion; } + inline const string& OldVersion() const { return m_oldVersion; } + + set<string> Files() const; + + void AddStringResource(const StringResource& res); + inline set<StringResource> const& GetStringResources() const { return m_strings; } + bool FindStringResource(const string& filename, int version, bool source); + + void Filter(bool (*func)(const string&,const TransUnit&,void*), void* cookie); + void Map(void (*func)(const string&,TransUnit*,void*), void* cookie); + + TransUnit* EditTransUnit(const string& file, const string& id); + + // exports this file as a n XMLNode, you own this object + XMLNode* ToXMLNode() const; + + // writes the ValuesFile out to a string in the canonical format (i.e. writes the contents of + // ToXMLNode()). + string ToString() const; + + Stats GetStats(const string& config) const; + +private: + struct File { + int Compare(const File& that) const; + + inline bool operator<(const File& that) const { return Compare(that) < 0; } + inline bool operator<=(const File& that) const { return Compare(that) <= 0; } + inline bool operator==(const File& that) const { return Compare(that) == 0; } + inline bool operator!=(const File& that) const { return Compare(that) != 0; } + inline bool operator>=(const File& that) const { return Compare(that) >= 0; } + inline bool operator>(const File& that) const { return Compare(that) > 0; } + + string filename; + vector<TransUnit> transUnits; + }; + + XLIFFFile(); + StringResource* find_string_res(TransUnit& g, const StringResource& str); + + Configuration m_sourceConfig; + Configuration m_targetConfig; + + string m_currentVersion; + string m_oldVersion; + + set<StringResource> m_strings; + vector<File> m_files; +}; + +int convert_html_to_xliff(const XMLNode* original, const string& name, XMLNode* addTo, int* phID); + +#endif // XLIFF_FILE_H diff --git a/tools/localize/XLIFFFile_test.cpp b/tools/localize/XLIFFFile_test.cpp new file mode 100644 index 0000000..52ed4c3 --- /dev/null +++ b/tools/localize/XLIFFFile_test.cpp @@ -0,0 +1,115 @@ +#include "XLIFFFile.h" +#include <stdio.h> +#include "ValuesFile.h" + +XMLNode* create_string_node(const StringResource& str, const string& name); + +static int +Parse_test() +{ + XLIFFFile* xf = XLIFFFile::Parse("testdata/xliff1.xliff"); + if (xf == NULL) { + return 1; + } + + set<StringResource> const& strings = xf->GetStringResources(); + + if (false) { + for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) { + const StringResource& str = *it; + printf("STRING!!! id=%s index=%d value='%s' pos=%s file=%s version=%d(%s)\n", + str.id.c_str(), str.index, + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.pos.ToString().c_str(), str.file.c_str(), str.version, + str.versionString.c_str()); + } + printf("XML:[[%s]]\n", xf->ToString().c_str()); + } + + delete xf; + return 0; +} + +static XMLNode* +add_html_tag(XMLNode* addTo, const string& tag) +{ + vector<XMLAttribute> attrs; + XMLNode* node = XMLNode::NewElement(GENERATED_POS, "", tag, attrs, XMLNode::EXACT); + addTo->EditChildren().push_back(node); + return node; +} + +static int +create_string_node_test() +{ + int err = 0; + StringResource res; + vector<XMLAttribute> attrs; + res.value = XMLNode::NewElement(GENERATED_POS, "", "something", attrs, XMLNode::EXACT); + res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " begin ", XMLNode::EXACT)); + + XMLNode* child; + + child = add_html_tag(res.value, "b"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT)); + + child = add_html_tag(res.value, "i"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "i", XMLNode::EXACT)); + + child = add_html_tag(child, "b"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "b", XMLNode::EXACT)); + + child = add_html_tag(res.value, "u"); + child->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, "u", XMLNode::EXACT)); + + + res.value->EditChildren().push_back(XMLNode::NewText(GENERATED_POS, " end ", XMLNode::EXACT)); + + XMLNode* xliff = create_string_node(res, "blah"); + + string oldString = res.value->ToString(XLIFF_NAMESPACES); + string newString = xliff->ToString(XLIFF_NAMESPACES); + + if (false) { + printf("OLD=\"%s\"\n", oldString.c_str()); + printf("NEW=\"%s\"\n", newString.c_str()); + } + + const char* const EXPECTED_OLD + = "<something> begin <b>b</b><i>i<b>b</b></i><u>u</u> end </something>"; + if (oldString != EXPECTED_OLD) { + fprintf(stderr, "oldString mismatch:\n"); + fprintf(stderr, " expected='%s'\n", EXPECTED_OLD); + fprintf(stderr, " actual='%s'\n", oldString.c_str()); + err |= 1; + } + + const char* const EXPECTED_NEW + = "<blah xml:space=\"preserve\"> begin <g ctype=\"bold\">b</g>" + "<g ctype=\"italic\">i<g ctype=\"bold\">b</g></g><g ctype=\"underline\">u</g>" + " end </blah>"; + if (newString != EXPECTED_NEW) { + fprintf(stderr, "newString mismatch:\n"); + fprintf(stderr, " expected='%s'\n", EXPECTED_NEW); + fprintf(stderr, " actual='%s'\n", newString.c_str()); + err |= 1; + } + + if (err != 0) { + fprintf(stderr, "create_string_node_test failed\n"); + } + return err; +} + +int +XLIFFFile_test() +{ + bool all = true; + int err = 0; + + if (all) err |= Parse_test(); + if (all) err |= create_string_node_test(); + + return err; +} + diff --git a/tools/localize/XMLHandler.cpp b/tools/localize/XMLHandler.cpp new file mode 100644 index 0000000..3fab211 --- /dev/null +++ b/tools/localize/XMLHandler.cpp @@ -0,0 +1,793 @@ +#include "XMLHandler.h" + +#include <algorithm> +#include <expat.h> +#include <stdio.h> +#include <string.h> +#include <fcntl.h> +#include <unistd.h> +#include <errno.h> + +#define NS_SEPARATOR 1 +#define MORE_INDENT " " + +static string +xml_text_escape(const string& s) +{ + string result; + const size_t N = s.length(); + for (size_t i=0; i<N; i++) { + char c = s[i]; + switch (c) { + case '<': + result += "<"; + break; + case '>': + result += ">"; + break; + case '&': + result += "&"; + break; + default: + result += c; + break; + } + } + return result; +} + +static string +xml_attr_escape(const string& s) +{ + string result; + const size_t N = s.length(); + for (size_t i=0; i<N; i++) { + char c = s[i]; + switch (c) { + case '\"': + result += """; + break; + default: + result += c; + break; + } + } + return result; +} + +XMLNamespaceMap::XMLNamespaceMap() +{ +} + +XMLNamespaceMap::XMLNamespaceMap(char const*const* nspaces) + +{ + while (*nspaces) { + m_map[nspaces[1]] = nspaces[0]; + nspaces += 2; + } +} + +string +XMLNamespaceMap::Get(const string& ns) const +{ + if (ns == "xml") { + return ns; + } + map<string,string>::const_iterator it = m_map.find(ns); + if (it == m_map.end()) { + return ""; + } else { + return it->second; + } +} + +string +XMLNamespaceMap::GetPrefix(const string& ns) const +{ + if (ns == "") { + return ""; + } + map<string,string>::const_iterator it = m_map.find(ns); + if (it != m_map.end()) { + if (it->second == "") { + return ""; + } else { + return it->second + ":"; + } + } else { + return ":"; // invalid + } +} + +void +XMLNamespaceMap::AddToAttributes(vector<XMLAttribute>* attrs) const +{ + map<string,string>::const_iterator it; + for (it=m_map.begin(); it!=m_map.end(); it++) { + if (it->second == "xml") { + continue; + } + XMLAttribute attr; + if (it->second == "") { + attr.name = "xmlns"; + } else { + attr.name = "xmlns:"; + attr.name += it->second; + } + attr.value = it->first; + attrs->push_back(attr); + } +} + +XMLAttribute::XMLAttribute() +{ +} + +XMLAttribute::XMLAttribute(const XMLAttribute& that) + :ns(that.ns), + name(that.name), + value(that.value) +{ +} + +XMLAttribute::XMLAttribute(string n, string na, string v) + :ns(n), + name(na), + value(v) +{ +} + +XMLAttribute::~XMLAttribute() +{ +} + +int +XMLAttribute::Compare(const XMLAttribute& that) const +{ + if (ns != that.ns) { + return ns < that.ns ? -1 : 1; + } + if (name != that.name) { + return name < that.name ? -1 : 1; + } + return 0; +} + +string +XMLAttribute::Find(const vector<XMLAttribute>& list, const string& ns, const string& name, + const string& def) +{ + const size_t N = list.size(); + for (size_t i=0; i<N; i++) { + const XMLAttribute& attr = list[i]; + if (attr.ns == ns && attr.name == name) { + return attr.value; + } + } + return def; +} + +struct xml_handler_data { + vector<XMLHandler*> stack; + XML_Parser parser; + vector<vector<XMLAttribute>*> attributes; + string filename; +}; + +XMLNode::XMLNode() +{ +} + +XMLNode::~XMLNode() +{ +// for_each(m_children.begin(), m_children.end(), delete_object<XMLNode>); +} + +XMLNode* +XMLNode::Clone() const +{ + switch (m_type) { + case ELEMENT: { + XMLNode* e = XMLNode::NewElement(m_pos, m_ns, m_name, m_attrs, m_pretty); + const size_t N = m_children.size(); + for (size_t i=0; i<N; i++) { + e->m_children.push_back(m_children[i]->Clone()); + } + return e; + } + case TEXT: { + return XMLNode::NewText(m_pos, m_text, m_pretty); + } + default: + return NULL; + } +} + +XMLNode* +XMLNode::NewElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, int pretty) +{ + XMLNode* node = new XMLNode(); + node->m_type = ELEMENT; + node->m_pretty = pretty; + node->m_pos = pos; + node->m_ns = ns; + node->m_name = name; + node->m_attrs = attrs; + return node; +} + +XMLNode* +XMLNode::NewText(const SourcePos& pos, const string& text, int pretty) +{ + XMLNode* node = new XMLNode(); + node->m_type = TEXT; + node->m_pretty = pretty; + node->m_pos = pos; + node->m_text = text; + return node; +} + +void +XMLNode::SetPrettyRecursive(int value) +{ + m_pretty = value; + const size_t N = m_children.size(); + for (size_t i=0; i<N; i++) { + m_children[i]->SetPrettyRecursive(value); + } +} + +string +XMLNode::ContentsToString(const XMLNamespaceMap& nspaces) const +{ + return contents_to_string(nspaces, ""); +} + +string +XMLNode::ToString(const XMLNamespaceMap& nspaces) const +{ + return to_string(nspaces, ""); +} + +string +XMLNode::OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const +{ + return open_tag_to_string(nspaces, "", pretty); +} + +string +XMLNode::contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const +{ + string result; + const size_t N = m_children.size(); + for (size_t i=0; i<N; i++) { + const XMLNode* child = m_children[i]; + switch (child->Type()) { + case ELEMENT: + if (m_pretty == PRETTY) { + result += '\n'; + result += indent; + } + case TEXT: + result += child->to_string(nspaces, indent); + break; + } + } + return result; +} + +string +trim_string(const string& str) +{ + const char* p = str.c_str(); + while (*p && isspace(*p)) { + p++; + } + const char* q = str.c_str() + str.length() - 1; + while (q > p && isspace(*q)) { + q--; + } + q++; + return string(p, q-p); +} + +string +XMLNode::open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent, int pretty) const +{ + if (m_type != ELEMENT) { + return ""; + } + string result = "<"; + result += nspaces.GetPrefix(m_ns); + result += m_name; + + vector<XMLAttribute> attrs = m_attrs; + + sort(attrs.begin(), attrs.end()); + + const size_t N = attrs.size(); + for (size_t i=0; i<N; i++) { + const XMLAttribute& attr = attrs[i]; + if (i == 0 || m_pretty == EXACT || pretty == EXACT) { + result += ' '; + } + else { + result += "\n"; + result += indent; + result += MORE_INDENT; + result += MORE_INDENT; + } + result += nspaces.GetPrefix(attr.ns); + result += attr.name; + result += "=\""; + result += xml_attr_escape(attr.value); + result += '\"'; + } + + if (m_children.size() > 0) { + result += '>'; + } else { + result += " />"; + } + return result; +} + +string +XMLNode::to_string(const XMLNamespaceMap& nspaces, const string& indent) const +{ + switch (m_type) + { + case TEXT: { + if (m_pretty == EXACT) { + return xml_text_escape(m_text); + } else { + return xml_text_escape(trim_string(m_text)); + } + } + case ELEMENT: { + string result = open_tag_to_string(nspaces, indent, PRETTY); + + if (m_children.size() > 0) { + result += contents_to_string(nspaces, indent + MORE_INDENT); + + if (m_pretty == PRETTY && m_children.size() > 0) { + result += '\n'; + result += indent; + } + + result += "</"; + result += nspaces.GetPrefix(m_ns); + result += m_name; + result += '>'; + } + return result; + } + default: + return ""; + } +} + +string +XMLNode::CollapseTextContents() const +{ + if (m_type == TEXT) { + return m_text; + } + else if (m_type == ELEMENT) { + string result; + + const size_t N=m_children.size(); + for (size_t i=0; i<N; i++) { + result += m_children[i]->CollapseTextContents(); + } + + return result; + } + else { + return ""; + } +} + +vector<XMLNode*> +XMLNode::GetElementsByName(const string& ns, const string& name) const +{ + vector<XMLNode*> result; + const size_t N=m_children.size(); + for (size_t i=0; i<N; i++) { + XMLNode* child = m_children[i]; + if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) { + result.push_back(child); + } + } + return result; +} + +XMLNode* +XMLNode::GetElementByNameAt(const string& ns, const string& name, size_t index) const +{ + vector<XMLNode*> result; + const size_t N=m_children.size(); + for (size_t i=0; i<N; i++) { + XMLNode* child = m_children[i]; + if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) { + if (index == 0) { + return child; + } else { + index--; + } + } + } + return NULL; +} + +size_t +XMLNode::CountElementsByName(const string& ns, const string& name) const +{ + size_t result = 0; + const size_t N=m_children.size(); + for (size_t i=0; i<N; i++) { + XMLNode* child = m_children[i]; + if (child->m_type == ELEMENT && child->m_ns == ns && child->m_name == name) { + result++; + } + } + return result; +} + +string +XMLNode::GetAttribute(const string& ns, const string& name, const string& def) const +{ + return XMLAttribute::Find(m_attrs, ns, name, def); +} + +static void +parse_namespace(const char* data, string* ns, string* name) +{ + const char* p = strchr(data, NS_SEPARATOR); + if (p != NULL) { + ns->assign(data, p-data); + name->assign(p+1); + } else { + ns->assign(""); + name->assign(data); + } +} + +static void +convert_attrs(const char** in, vector<XMLAttribute>* out) +{ + while (*in) { + XMLAttribute attr; + parse_namespace(in[0], &attr.ns, &attr.name); + attr.value = in[1]; + out->push_back(attr); + in += 2; + } +} + +static bool +list_contains(const vector<XMLHandler*>& stack, XMLHandler* handler) +{ + const size_t N = stack.size(); + for (size_t i=0; i<N; i++) { + if (stack[i] == handler) { + return true; + } + } + return false; +} + +static void XMLCALL +start_element_handler(void *userData, const char *name, const char **attrs) +{ + xml_handler_data* data = (xml_handler_data*)userData; + + XMLHandler* handler = data->stack[data->stack.size()-1]; + + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + string nsString; + string nameString; + XMLHandler* next = handler; + vector<XMLAttribute> attributes; + + parse_namespace(name, &nsString, &nameString); + convert_attrs(attrs, &attributes); + + handler->OnStartElement(pos, nsString, nameString, attributes, &next); + + if (next == NULL) { + next = handler; + } + + if (next != handler) { + next->elementPos = pos; + next->elementNamespace = nsString; + next->elementName = nameString; + next->elementAttributes = attributes; + } + + data->stack.push_back(next); +} + +static void XMLCALL +end_element_handler(void *userData, const char *name) +{ + xml_handler_data* data = (xml_handler_data*)userData; + + XMLHandler* handler = data->stack[data->stack.size()-1]; + data->stack.pop_back(); + + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + + if (!list_contains(data->stack, handler)) { + handler->OnDone(pos); + if (data->stack.size() > 1) { + // not top one + delete handler; + } + } + + handler = data->stack[data->stack.size()-1]; + + string nsString; + string nameString; + + parse_namespace(name, &nsString, &nameString); + + handler->OnEndElement(pos, nsString, nameString); +} + +static void XMLCALL +text_handler(void *userData, const XML_Char *s, int len) +{ + xml_handler_data* data = (xml_handler_data*)userData; + XMLHandler* handler = data->stack[data->stack.size()-1]; + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + handler->OnText(pos, string(s, len)); +} + +static void XMLCALL +comment_handler(void *userData, const char *comment) +{ + xml_handler_data* data = (xml_handler_data*)userData; + XMLHandler* handler = data->stack[data->stack.size()-1]; + SourcePos pos(data->filename, (int)XML_GetCurrentLineNumber(data->parser)); + handler->OnComment(pos, string(comment)); +} + +bool +XMLHandler::ParseFile(const string& filename, XMLHandler* handler) +{ + char buf[16384]; + int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) { + SourcePos(filename, -1).Error("Unable to open file for read: %s", strerror(errno)); + return false; + } + + XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR); + xml_handler_data state; + state.stack.push_back(handler); + state.parser = parser; + state.filename = filename; + + XML_SetUserData(parser, &state); + XML_SetElementHandler(parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler(parser, text_handler); + XML_SetCommentHandler(parser, comment_handler); + + ssize_t len; + bool done; + do { + len = read(fd, buf, sizeof(buf)); + done = len < (ssize_t)sizeof(buf); + if (len < 0) { + SourcePos(filename, -1).Error("Error reading file: %s\n", strerror(errno)); + close(fd); + return false; + } + if (XML_Parse(parser, buf, len, done) == XML_STATUS_ERROR) { + SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error( + "Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser))); + close(fd); + return false; + } + } while (!done); + + XML_ParserFree(parser); + + close(fd); + + return true; +} + +bool +XMLHandler::ParseString(const string& filename, const string& text, XMLHandler* handler) +{ + XML_Parser parser = XML_ParserCreateNS(NULL, NS_SEPARATOR); + xml_handler_data state; + state.stack.push_back(handler); + state.parser = parser; + state.filename = filename; + + XML_SetUserData(parser, &state); + XML_SetElementHandler(parser, start_element_handler, end_element_handler); + XML_SetCharacterDataHandler(parser, text_handler); + XML_SetCommentHandler(parser, comment_handler); + + if (XML_Parse(parser, text.c_str(), text.size(), true) == XML_STATUS_ERROR) { + SourcePos(filename, (int)XML_GetCurrentLineNumber(parser)).Error( + "Error parsing XML: %s\n", XML_ErrorString(XML_GetErrorCode(parser))); + return false; + } + + XML_ParserFree(parser); + + return true; +} + +XMLHandler::XMLHandler() +{ +} + +XMLHandler::~XMLHandler() +{ +} + +int +XMLHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next) +{ + return 0; +} + +int +XMLHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name) +{ + return 0; +} + +int +XMLHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +XMLHandler::OnComment(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +XMLHandler::OnDone(const SourcePos& pos) +{ + return 0; +} + +TopElementHandler::TopElementHandler(const string& ns, const string& name, XMLHandler* next) + :m_ns(ns), + m_name(name), + m_next(next) +{ +} + +int +TopElementHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next) +{ + *next = m_next; + return 0; +} + +int +TopElementHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name) +{ + return 0; +} + +int +TopElementHandler::OnText(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +TopElementHandler::OnDone(const SourcePos& pos) +{ + return 0; +} + + +NodeHandler::NodeHandler(XMLNode* root, int pretty) + :m_root(root), + m_pretty(pretty) +{ + if (root != NULL) { + m_nodes.push_back(root); + } +} + +NodeHandler::~NodeHandler() +{ +} + +int +NodeHandler::OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next) +{ + int pretty; + if (XMLAttribute::Find(attrs, XMLNS_XMLNS, "space", "") == "preserve") { + pretty = XMLNode::EXACT; + } else { + if (m_root == NULL) { + pretty = m_pretty; + } else { + pretty = m_nodes[m_nodes.size()-1]->Pretty(); + } + } + XMLNode* n = XMLNode::NewElement(pos, ns, name, attrs, pretty); + if (m_root == NULL) { + m_root = n; + } else { + m_nodes[m_nodes.size()-1]->EditChildren().push_back(n); + } + m_nodes.push_back(n); + return 0; +} + +int +NodeHandler::OnEndElement(const SourcePos& pos, const string& ns, const string& name) +{ + m_nodes.pop_back(); + return 0; +} + +int +NodeHandler::OnText(const SourcePos& pos, const string& text) +{ + if (m_root == NULL) { + return 1; + } + XMLNode* n = XMLNode::NewText(pos, text, m_nodes[m_nodes.size()-1]->Pretty()); + m_nodes[m_nodes.size()-1]->EditChildren().push_back(n); + return 0; +} + +int +NodeHandler::OnComment(const SourcePos& pos, const string& text) +{ + return 0; +} + +int +NodeHandler::OnDone(const SourcePos& pos) +{ + return 0; +} + +XMLNode* +NodeHandler::ParseFile(const string& filename, int pretty) +{ + NodeHandler handler(NULL, pretty); + if (!XMLHandler::ParseFile(filename, &handler)) { + fprintf(stderr, "error parsing file: %s\n", filename.c_str()); + return NULL; + } + return handler.Root(); +} + +XMLNode* +NodeHandler::ParseString(const string& filename, const string& text, int pretty) +{ + NodeHandler handler(NULL, pretty); + if (!XMLHandler::ParseString(filename, text, &handler)) { + fprintf(stderr, "error parsing file: %s\n", filename.c_str()); + return NULL; + } + return handler.Root(); +} + + diff --git a/tools/localize/XMLHandler.h b/tools/localize/XMLHandler.h new file mode 100644 index 0000000..1130710 --- /dev/null +++ b/tools/localize/XMLHandler.h @@ -0,0 +1,197 @@ +#ifndef XML_H +#define XML_H + +#include "SourcePos.h" + +#include <string> +#include <vector> +#include <map> + +#define XMLNS_XMLNS "http://www.w3.org/XML/1998/namespace" + +using namespace std; + +string trim_string(const string& str); + +struct XMLAttribute +{ + string ns; + string name; + string value; + + XMLAttribute(); + XMLAttribute(const XMLAttribute& that); + XMLAttribute(string ns, string name, string value); + ~XMLAttribute(); + + int Compare(const XMLAttribute& that) const; + + inline bool operator<(const XMLAttribute& that) const { return Compare(that) < 0; } + inline bool operator<=(const XMLAttribute& that) const { return Compare(that) <= 0; } + inline bool operator==(const XMLAttribute& that) const { return Compare(that) == 0; } + inline bool operator!=(const XMLAttribute& that) const { return Compare(that) != 0; } + inline bool operator>=(const XMLAttribute& that) const { return Compare(that) >= 0; } + inline bool operator>(const XMLAttribute& that) const { return Compare(that) > 0; } + + static string Find(const vector<XMLAttribute>& list, + const string& ns, const string& name, const string& def); +}; + +class XMLNamespaceMap +{ +public: + XMLNamespaceMap(); + XMLNamespaceMap(char const*const* nspaces); + string Get(const string& ns) const; + string GetPrefix(const string& ns) const; + void AddToAttributes(vector<XMLAttribute>* attrs) const; +private: + map<string,string> m_map; +}; + +struct XMLNode +{ +public: + enum { + EXACT = 0, + PRETTY = 1 + }; + + enum { + ELEMENT = 0, + TEXT = 1 + }; + + static XMLNode* NewElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, int pretty); + static XMLNode* NewText(const SourcePos& pos, const string& text, int pretty); + + ~XMLNode(); + + // a deep copy + XMLNode* Clone() const; + + inline int Type() const { return m_type; } + inline int Pretty() const { return m_pretty; } + void SetPrettyRecursive(int value); + string ContentsToString(const XMLNamespaceMap& nspaces) const; + string ToString(const XMLNamespaceMap& nspaces) const; + string OpenTagToString(const XMLNamespaceMap& nspaces, int pretty) const; + + string CollapseTextContents() const; + + inline const SourcePos& Position() const { return m_pos; } + + // element + inline string Namespace() const { return m_ns; } + inline string Name() const { return m_name; } + inline void SetName(const string& ns, const string& n) { m_ns = ns; m_name = n; } + inline const vector<XMLAttribute>& Attributes() const { return m_attrs; } + inline vector<XMLAttribute>& EditAttributes() { return m_attrs; } + inline const vector<XMLNode*>& Children() const { return m_children; } + inline vector<XMLNode*>& EditChildren() { return m_children; } + vector<XMLNode*> GetElementsByName(const string& ns, const string& name) const; + XMLNode* GetElementByNameAt(const string& ns, const string& name, size_t index) const; + size_t CountElementsByName(const string& ns, const string& name) const; + string GetAttribute(const string& ns, const string& name, const string& def) const; + + // text + inline string Text() const { return m_text; } + +private: + XMLNode(); + XMLNode(const XMLNode&); + + string contents_to_string(const XMLNamespaceMap& nspaces, const string& indent) const; + string to_string(const XMLNamespaceMap& nspaces, const string& indent) const; + string open_tag_to_string(const XMLNamespaceMap& nspaces, const string& indent, + int pretty) const; + + int m_type; + int m_pretty; + SourcePos m_pos; + + // element + string m_ns; + string m_name; + vector<XMLAttribute> m_attrs; + vector<XMLNode*> m_children; + + // text + string m_text; +}; + +class XMLHandler +{ +public: + // information about the element that started us + SourcePos elementPos; + string elementNamespace; + string elementName; + vector<XMLAttribute> elementAttributes; + + XMLHandler(); + virtual ~XMLHandler(); + + XMLHandler* parent; + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next); + virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + virtual int OnDone(const SourcePos& pos); + + static bool ParseFile(const string& filename, XMLHandler* handler); + static bool ParseString(const string& filename, const string& text, XMLHandler* handler); +}; + +class TopElementHandler : public XMLHandler +{ +public: + TopElementHandler(const string& ns, const string& name, XMLHandler* next); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next); + virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnDone(const SourcePos& endPos); + +private: + string m_ns; + string m_name; + XMLHandler* m_next; +}; + +class NodeHandler : public XMLHandler +{ +public: + // after it's done, you own everything created and added to root + NodeHandler(XMLNode* root, int pretty); + ~NodeHandler(); + + virtual int OnStartElement(const SourcePos& pos, const string& ns, const string& name, + const vector<XMLAttribute>& attrs, XMLHandler** next); + virtual int OnEndElement(const SourcePos& pos, const string& ns, const string& name); + virtual int OnText(const SourcePos& pos, const string& text); + virtual int OnComment(const SourcePos& pos, const string& text); + virtual int OnDone(const SourcePos& endPos); + + inline XMLNode* Root() const { return m_root; } + + static XMLNode* ParseFile(const string& filename, int pretty); + static XMLNode* ParseString(const string& filename, const string& text, int pretty); + +private: + XMLNode* m_root; + int m_pretty; + vector<XMLNode*> m_nodes; +}; + +template <class T> +static void delete_object(T* obj) +{ + delete obj; +} + +#endif // XML_H diff --git a/tools/localize/XMLHandler_test.cpp b/tools/localize/XMLHandler_test.cpp new file mode 100644 index 0000000..1c81c0c --- /dev/null +++ b/tools/localize/XMLHandler_test.cpp @@ -0,0 +1,133 @@ +#include "XMLHandler.h" +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> + +const char *const NS_MAP[] = { + "xml", XMLNS_XMLNS, + NULL, NULL +}; + +const XMLNamespaceMap NO_NAMESPACES(NS_MAP); + +char const*const EXPECTED_EXACT = + "<ASDF>\n" + " <a id=\"system\" old-cl=\"1\" new-cl=\"43019\">\n" + " <app dir=\"apps/common\" />\n" + " </a>\n" + " <a id=\"samples\" old-cl=\"1\" new-cl=\"43019\">asdf\n" + " <app dir=\"samples/NotePad\" />\n" + " <app dir=\"samples/LunarLander\" />\n" + " <something>a<b>,</b>b </something>\n" + " <exact xml:space=\"preserve\">a<b>,</b>b </exact>\n" + " </a>\n" + "</ASDF>\n"; + +char const*const EXPECTED_PRETTY = + "<ASDF>\n" + " <a id=\"system\"\n" + " old-cl=\"1\"\n" + " new-cl=\"43019\">\n" + " <app dir=\"apps/common\" />\n" + " </a>\n" + " <a id=\"samples\"\n" + " old-cl=\"1\"\n" + " new-cl=\"43019\">asdf\n" + " <app dir=\"samples/NotePad\" />\n" + " <app dir=\"samples/LunarLander\" />\n" + " <something>a\n" + " <b>,\n" + " </b>b \n" + " </something>\n" + " <exact xml:space=\"preserve\">a<b>,</b>b </exact>\n" + " </a>\n" + "</ASDF>\n"; + +static string +read_file(const string& filename) +{ + char buf[1024]; + int fd = open(filename.c_str(), O_RDONLY); + if (fd < 0) { + return ""; + } + string result; + while (true) { + ssize_t len = read(fd, buf, sizeof(buf)-1); + buf[len] = '\0'; + if (len <= 0) { + break; + } + result.append(buf, len); + } + close(fd); + return result; +} + +static int +ParseFile_EXACT_test() +{ + XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::EXACT); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_EXACT; +} + +static int +ParseFile_PRETTY_test() +{ + XMLNode* root = NodeHandler::ParseFile("testdata/xml.xml", XMLNode::PRETTY); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_PRETTY; +} + +static int +ParseString_EXACT_test() +{ + string text = read_file("testdata/xml.xml"); + XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::EXACT); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_EXACT; +} + +static int +ParseString_PRETTY_test() +{ + string text = read_file("testdata/xml.xml"); + XMLNode* root = NodeHandler::ParseString("testdata/xml.xml", text, XMLNode::PRETTY); + if (root == NULL) { + return 1; + } + string result = root->ToString(NO_NAMESPACES); + delete root; + //printf("[[%s]]\n", result.c_str()); + return result == EXPECTED_PRETTY; +} + +int +XMLHandler_test() +{ + int err = 0; + bool all = true; + + if (all) err |= ParseFile_EXACT_test(); + if (all) err |= ParseFile_PRETTY_test(); + if (all) err |= ParseString_EXACT_test(); + if (all) err |= ParseString_PRETTY_test(); + + return err; +} diff --git a/tools/localize/XMLNode.h b/tools/localize/XMLNode.h new file mode 100644 index 0000000..bfb9f55 --- /dev/null +++ b/tools/localize/XMLNode.h @@ -0,0 +1,19 @@ +#ifndef XMLNODE_H +#define XMLNODE_H + +#include <string> + +using namespace std; + +struct XMLAttribute +{ + string ns; + string name; + string value; + + static string Find(const vector<XMLAttribute>& list, + const string& ns, const string& name, const string& def); +}; + + +#endif // XMLNODE_H diff --git a/tools/localize/file_utils.cpp b/tools/localize/file_utils.cpp new file mode 100644 index 0000000..bb82a9c --- /dev/null +++ b/tools/localize/file_utils.cpp @@ -0,0 +1,143 @@ +#include <string.h> +#include <stdlib.h> +#include <unistd.h> +#include "file_utils.h" +#include "Perforce.h" +#include <sys/fcntl.h> +#include <sys/stat.h> +#include <errno.h> +#include <host/Directories.h> +#include "log.h" + +string +translated_file_name(const string& file, const string& locale) +{ + const char* str = file.c_str(); + const char* p = str + file.length(); + const char* rest = NULL; + const char* values = p; + + while (p > str) { + p--; + if (*p == '/') { + rest = values; + values = p; + if (0 == strncmp("values", values+1, rest-values-1)) { + break; + } + } + } + values++; + + string result(str, values-str); + result.append(values, rest-values); + + string language, region; + if (locale == "") { + language = ""; + region = ""; + } + else if (!split_locale(locale, &language, ®ion)) { + return ""; + } + + if (language != "") { + result += '-'; + result += language; + } + if (region != "") { + result += "-r"; + result += region; + } + + result += rest; + + return result; +} + +ValuesFile* +get_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure) +{ + int err; + string text; + + log_printf("get_values_file filename=%s\n", filename.c_str()); + err = Perforce::GetFile(filename, versionString, &text, printOnFailure); + if (err != 0 || text == "") { + return NULL; + } + + ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version, + versionString); + if (result == NULL) { + fprintf(stderr, "unable to parse file: %s\n", filename.c_str()); + exit(1); + } + return result; +} + +ValuesFile* +get_local_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure) +{ + int err; + string text; + char buf[2049]; + int fd; + ssize_t amt; + + fd = open(filename.c_str(), O_RDONLY); + if (fd == -1) { + fprintf(stderr, "unable to open file: %s\n", filename.c_str()); + return NULL; + } + + while ((amt = read(fd, buf, sizeof(buf)-1)) > 0) { + text.append(buf, amt); + } + + close(fd); + + if (text == "") { + return NULL; + } + + ValuesFile* result = ValuesFile::ParseString(filename, text, configuration, version, + versionString); + if (result == NULL) { + fprintf(stderr, "unable to parse file: %s\n", filename.c_str()); + exit(1); + } + return result; +} + +void +print_file_status(size_t j, size_t J, const string& message) +{ + printf("\r%s file %zd of %zd...", message.c_str(), j, J); + fflush(stdout); +} + +int +write_to_file(const string& filename, const string& text) +{ + mkdirs(parent_dir(filename).c_str()); + int fd = open(filename.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); + if (fd < 0) { + fprintf(stderr, "unable to open file for write (%s): %s\n", strerror(errno), + filename.c_str()); + return -1; + } + + ssize_t amt = write(fd, text.c_str(), text.length()); + + close(fd); + + if (amt < 0) { + return amt; + } + return amt == (ssize_t)text.length() ? 0 : -1; +} + + diff --git a/tools/localize/file_utils.h b/tools/localize/file_utils.h new file mode 100644 index 0000000..3b3fa21 --- /dev/null +++ b/tools/localize/file_utils.h @@ -0,0 +1,21 @@ +#ifndef FILE_UTILS_H +#define FILE_UTILS_H + +#include "ValuesFile.h" +#include "Configuration.h" +#include <string> + +using namespace std; + +string translated_file_name(const string& file, const string& locale); + +ValuesFile* get_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure); +ValuesFile* get_local_values_file(const string& filename, const Configuration& configuration, + int version, const string& versionString, bool printOnFailure); + +void print_file_status(size_t j, size_t J, const string& message = "Reading"); +int write_to_file(const string& filename, const string& text); + + +#endif // FILE_UTILS_H diff --git a/tools/localize/localize.cpp b/tools/localize/localize.cpp new file mode 100644 index 0000000..c0d84cc --- /dev/null +++ b/tools/localize/localize.cpp @@ -0,0 +1,767 @@ +#include "SourcePos.h" +#include "ValuesFile.h" +#include "XLIFFFile.h" +#include "Perforce.h" +#include "merge_res_and_xliff.h" +#include "localize.h" +#include "file_utils.h" +#include "res_check.h" +#include "xmb.h" + +#include <host/pseudolocalize.h> + +#include <stdlib.h> +#include <stdarg.h> +#include <sstream> +#include <stdio.h> +#include <string.h> + +using namespace std; + +FILE* g_logFile = NULL; + +int test(); + +int +read_settings(const string& filename, map<string,Settings>* result, const string& rootDir) +{ + XMLNode* root = NodeHandler::ParseFile(filename, XMLNode::PRETTY); + if (root == NULL) { + SourcePos(filename, -1).Error("Error reading file."); + return 1; + } + + // <configuration> + vector<XMLNode*> configNodes = root->GetElementsByName("", "configuration"); + const size_t I = configNodes.size(); + for (size_t i=0; i<I; i++) { + const XMLNode* configNode = configNodes[i]; + + Settings settings; + settings.id = configNode->GetAttribute("", "id", ""); + if (settings.id == "") { + configNode->Position().Error("<configuration> needs an id attribute."); + delete root; + return 1; + } + + settings.oldVersion = configNode->GetAttribute("", "old-cl", ""); + + settings.currentVersion = configNode->GetAttribute("", "new-cl", ""); + if (settings.currentVersion == "") { + configNode->Position().Error("<configuration> needs a new-cl attribute."); + delete root; + return 1; + } + + // <app> + vector<XMLNode*> appNodes = configNode->GetElementsByName("", "app"); + + const size_t J = appNodes.size(); + for (size_t j=0; j<J; j++) { + const XMLNode* appNode = appNodes[j]; + + string dir = appNode->GetAttribute("", "dir", ""); + if (dir == "") { + appNode->Position().Error("<app> needs a dir attribute."); + delete root; + return 1; + } + + settings.apps.push_back(dir); + } + + // <reject> + vector<XMLNode*> rejectNodes = configNode->GetElementsByName("", "reject"); + + const size_t K = rejectNodes.size(); + for (size_t k=0; k<K; k++) { + const XMLNode* rejectNode = rejectNodes[k]; + + Reject reject; + + reject.file = rejectNode->GetAttribute("", "file", ""); + if (reject.file == "") { + rejectNode->Position().Error("<reject> needs a file attribute."); + delete root; + return 1; + } + string f = reject.file; + reject.file = rootDir; + reject.file += '/'; + reject.file += f; + + reject.name = rejectNode->GetAttribute("", "name", ""); + if (reject.name == "") { + rejectNode->Position().Error("<reject> needs a name attribute."); + delete root; + return 1; + } + + reject.comment = trim_string(rejectNode->CollapseTextContents()); + + settings.reject.push_back(reject); + } + + (*result)[settings.id] = settings; + } + + delete root; + return 0; +} + + +static void +ValuesFile_to_XLIFFFile(const ValuesFile* values, XLIFFFile* xliff, const string& englishFilename) +{ + const set<StringResource>& strings = values->GetStrings(); + for (set<StringResource>::const_iterator it=strings.begin(); it!=strings.end(); it++) { + StringResource res = *it; + res.file = englishFilename; + xliff->AddStringResource(res); + } +} + +static bool +contains_reject(const Settings& settings, const string& file, const TransUnit& tu) +{ + const string name = tu.id; + const vector<Reject>& reject = settings.reject; + const size_t I = reject.size(); + for (size_t i=0; i<I; i++) { + const Reject& r = reject[i]; + if (r.file == file && r.name == name) { + return true; + } + } + return false; +} + +/** + * If it's been rejected, then we keep whatever info we have. + * + * Implements this truth table: + * + * S AT AS Keep + * ----------------------- + * 0 0 0 0 (this case can't happen) + * 0 0 1 0 (it was there, never translated, and removed) + * 0 1 0 0 (somehow it got translated, but it was removed) + * 0 1 1 0 (it was removed after having been translated) + * + * 1 0 0 1 (it was just added) + * 1 0 1 1 (it was added, has been changed, but it never got translated) + * 1 1 0 1 (somehow it got translated, but we don't know based on what) + * 1 1 1 0/1 (it's in both. 0 if S=AS b/c there's no need to retranslate if they're + * the same. 1 if S!=AS because S changed, so it should be retranslated) + * + * The first four are cases where, whatever happened in the past, the string isn't there + * now, so it shouldn't be in the XLIFF file. + * + * For cases 4 and 5, the string has never been translated, so get it translated. + * + * For case 6, it's unclear where the translated version came from, so we're conservative + * and send it back for them to have another shot at. + * + * For case 7, we have some data. We have two choices. We could rely on the translator's + * translation memory or tools to notice that the strings haven't changed, and populate the + * <target> field themselves. Or if the string hasn't changed since last time, we can just + * not even tell them about it. As the project nears the end, it will be convenient to see + * the xliff files reducing in size, so we pick the latter. Obviously, if the string has + * changed, then we need to get it retranslated. + */ +bool +keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie) +{ + const Settings* settings = reinterpret_cast<const Settings*>(cookie); + + if (contains_reject(*settings, file, unit)) { + return true; + } + + if (unit.source.id == "") { + return false; + } + if (unit.altTarget.id == "" || unit.altSource.id == "") { + return true; + } + return unit.source.value->ContentsToString(XLIFF_NAMESPACES) + != unit.altSource.value->ContentsToString(XLIFF_NAMESPACES); +} + +int +validate_config(const string& settingsFile, const map<string,Settings>& settings, + const string& config) +{ + if (settings.find(config) == settings.end()) { + SourcePos(settingsFile, -1).Error("settings file does not contain setting: %s\n", + config.c_str()); + return 1; + } + return 0; +} + +int +validate_configs(const string& settingsFile, const map<string,Settings>& settings, + const vector<string>& configs) +{ + int err = 0; + for (size_t i=0; i<configs.size(); i++) { + string config = configs[i]; + err |= validate_config(settingsFile, settings, config); + } + return err; +} + +int +select_files(vector<string> *resFiles, const string& config, + const map<string,Settings>& settings, const string& rootDir) +{ + int err; + vector<vector<string> > allResFiles; + vector<string> configs; + configs.push_back(config); + err = select_files(&allResFiles, configs, settings, rootDir); + if (err == 0) { + *resFiles = allResFiles[0]; + } + return err; +} + +int +select_files(vector<vector<string> > *allResFiles, const vector<string>& configs, + const map<string,Settings>& settings, const string& rootDir) +{ + int err; + printf("Selecting files..."); + fflush(stdout); + + for (size_t i=0; i<configs.size(); i++) { + const string& config = configs[i]; + const Settings& setting = settings.find(config)->second; + + vector<string> resFiles; + err = Perforce::GetResourceFileNames(setting.currentVersion, rootDir, + setting.apps, &resFiles, true); + if (err != 0) { + fprintf(stderr, "error with perforce. bailing\n"); + return err; + } + + allResFiles->push_back(resFiles); + } + return 0; +} + +static int +do_export(const string& settingsFile, const string& rootDir, const string& outDir, + const string& targetLocale, const vector<string>& configs) +{ + bool success = true; + int err; + + if (false) { + printf("settingsFile=%s\n", settingsFile.c_str()); + printf("rootDir=%s\n", rootDir.c_str()); + printf("outDir=%s\n", outDir.c_str()); + for (size_t i=0; i<configs.size(); i++) { + printf("config[%zd]=%s\n", i, configs[i].c_str()); + } + } + + map<string,Settings> settings; + err = read_settings(settingsFile, &settings, rootDir); + if (err != 0) { + return err; + } + + err = validate_configs(settingsFile, settings, configs); + if (err != 0) { + return err; + } + + vector<vector<string> > allResFiles; + err = select_files(&allResFiles, configs, settings, rootDir); + if (err != 0) { + return err; + } + + size_t totalFileCount = 0; + for (size_t i=0; i<allResFiles.size(); i++) { + totalFileCount += allResFiles[i].size(); + } + totalFileCount *= 3; // we try all 3 versions of the file + + size_t fileProgress = 0; + vector<Stats> stats; + vector<pair<string,XLIFFFile*> > xliffs; + + for (size_t i=0; i<configs.size(); i++) { + const string& config = configs[i]; + const Settings& setting = settings[config]; + + if (false) { + fprintf(stderr, "Configuration: %s (%zd of %zd)\n", config.c_str(), i+1, + configs.size()); + fprintf(stderr, " Old CL: %s\n", setting.oldVersion.c_str()); + fprintf(stderr, " Current CL: %s\n", setting.currentVersion.c_str()); + } + + Configuration english; + english.locale = "en_US"; + Configuration translated; + translated.locale = targetLocale; + XLIFFFile* xliff = XLIFFFile::Create(english, translated, setting.currentVersion); + + const vector<string>& resFiles = allResFiles[i]; + const size_t J = resFiles.size(); + for (size_t j=0; j<J; j++) { + string resFile = resFiles[j]; + + // parse the files into a ValuesFile + // pull out the strings and add them to the XLIFFFile + + // current file + print_file_status(++fileProgress, totalFileCount); + ValuesFile* currentFile = get_values_file(resFile, english, CURRENT_VERSION, + setting.currentVersion, true); + if (currentFile != NULL) { + ValuesFile_to_XLIFFFile(currentFile, xliff, resFile); + //printf("currentFile=[%s]\n", currentFile->ToString().c_str()); + } else { + fprintf(stderr, "error reading file %s@%s\n", resFile.c_str(), + setting.currentVersion.c_str()); + success = false; + } + + // old file + print_file_status(++fileProgress, totalFileCount); + ValuesFile* oldFile = get_values_file(resFile, english, OLD_VERSION, + setting.oldVersion, false); + if (oldFile != NULL) { + ValuesFile_to_XLIFFFile(oldFile, xliff, resFile); + //printf("oldFile=[%s]\n", oldFile->ToString().c_str()); + } + + // translated version + // (get the head of the tree for the most recent translation, but it's considered + // the old one because the "current" one hasn't been made yet, and this goes into + // the <alt-trans> tag if necessary + print_file_status(++fileProgress, totalFileCount); + string transFilename = translated_file_name(resFile, targetLocale); + ValuesFile* transFile = get_values_file(transFilename, translated, OLD_VERSION, + setting.currentVersion, false); + if (transFile != NULL) { + ValuesFile_to_XLIFFFile(transFile, xliff, resFile); + } + + delete currentFile; + delete oldFile; + delete transFile; + } + + Stats beforeFilterStats = xliff->GetStats(config); + + // run through the XLIFFFile and strip out TransUnits that have identical + // old and current source values and are not in the reject list, or just + // old values and no source values + xliff->Filter(keep_this_trans_unit, (void*)&setting); + + Stats afterFilterStats = xliff->GetStats(config); + afterFilterStats.totalStrings = beforeFilterStats.totalStrings; + + // add the reject comments + for (vector<Reject>::const_iterator reject = setting.reject.begin(); + reject != setting.reject.end(); reject++) { + TransUnit* tu = xliff->EditTransUnit(reject->file, reject->name); + tu->rejectComment = reject->comment; + } + + // config-locale-current_cl.xliff + stringstream filename; + if (outDir != "") { + filename << outDir << '/'; + } + filename << config << '-' << targetLocale << '-' << setting.currentVersion << ".xliff"; + xliffs.push_back(pair<string,XLIFFFile*>(filename.str(), xliff)); + + stats.push_back(afterFilterStats); + } + + // today is a good day to die + if (!success || SourcePos::HasErrors()) { + return 1; + } + + // write the XLIFF files + printf("\nWriting %zd file%s...\n", xliffs.size(), xliffs.size() == 1 ? "" : "s"); + for (vector<pair<string,XLIFFFile*> >::iterator it = xliffs.begin(); it != xliffs.end(); it++) { + const string& filename = it->first; + XLIFFFile* xliff = it->second; + string text = xliff->ToString(); + write_to_file(filename, text); + } + + // the stats + printf("\n" + " to without total\n" + " config files translate comments strings\n" + "-----------------------------------------------------------------------\n"); + Stats totals; + totals.config = "total"; + totals.files = 0; + totals.toBeTranslated = 0; + totals.noComments = 0; + totals.totalStrings = 0; + for (vector<Stats>::iterator it=stats.begin(); it!=stats.end(); it++) { + string cfg = it->config; + if (cfg.length() > 20) { + cfg.resize(20); + } + printf(" %-20s %-9zd %-9zd %-9zd %-19zd\n", cfg.c_str(), it->files, + it->toBeTranslated, it->noComments, it->totalStrings); + totals.files += it->files; + totals.toBeTranslated += it->toBeTranslated; + totals.noComments += it->noComments; + totals.totalStrings += it->totalStrings; + } + if (stats.size() > 1) { + printf("-----------------------------------------------------------------------\n" + " %-20s %-9zd %-9zd %-9zd %-19zd\n", totals.config.c_str(), totals.files, + totals.toBeTranslated, totals.noComments, totals.totalStrings); + } + printf("\n"); + return 0; +} + +struct PseudolocalizeSettings { + XLIFFFile* xliff; + bool expand; +}; + + +string +pseudolocalize_string(const string& source, const PseudolocalizeSettings* settings) +{ + return pseudolocalize_string(source); +} + +static XMLNode* +pseudolocalize_xml_node(const XMLNode* source, const PseudolocalizeSettings* settings) +{ + if (source->Type() == XMLNode::TEXT) { + return XMLNode::NewText(source->Position(), pseudolocalize_string(source->Text(), settings), + source->Pretty()); + } else { + XMLNode* target; + if (source->Namespace() == XLIFF_XMLNS && source->Name() == "g") { + // XXX don't translate these + target = XMLNode::NewElement(source->Position(), source->Namespace(), + source->Name(), source->Attributes(), source->Pretty()); + } else { + target = XMLNode::NewElement(source->Position(), source->Namespace(), + source->Name(), source->Attributes(), source->Pretty()); + } + + const vector<XMLNode*>& children = source->Children(); + const size_t I = children.size(); + for (size_t i=0; i<I; i++) { + target->EditChildren().push_back(pseudolocalize_xml_node(children[i], settings)); + } + + return target; + } +} + +void +pseudolocalize_trans_unit(const string&file, TransUnit* unit, void* cookie) +{ + const PseudolocalizeSettings* settings = (PseudolocalizeSettings*)cookie; + + const StringResource& source = unit->source; + StringResource* target = &unit->target; + *target = source; + + target->config = settings->xliff->TargetConfig(); + + delete target->value; + target->value = pseudolocalize_xml_node(source.value, settings); +} + +int +pseudolocalize_xliff(XLIFFFile* xliff, bool expand) +{ + PseudolocalizeSettings settings; + + settings.xliff = xliff; + settings.expand = expand; + xliff->Map(pseudolocalize_trans_unit, &settings); + return 0; +} + +static int +do_pseudo(const string& infile, const string& outfile, bool expand) +{ + int err; + + XLIFFFile* xliff = XLIFFFile::Parse(infile); + if (xliff == NULL) { + return 1; + } + + pseudolocalize_xliff(xliff, expand); + + err = write_to_file(outfile, xliff->ToString()); + + delete xliff; + + return err; +} + +void +log_printf(const char *fmt, ...) +{ + int ret; + va_list ap; + + if (g_logFile != NULL) { + va_start(ap, fmt); + ret = vfprintf(g_logFile, fmt, ap); + va_end(ap); + fflush(g_logFile); + } +} + +void +close_log_file() +{ + if (g_logFile != NULL) { + fclose(g_logFile); + } +} + +void +open_log_file(const char* file) +{ + g_logFile = fopen(file, "w"); + printf("log file: %s -- %p\n", file, g_logFile); + atexit(close_log_file); +} + +static int +usage() +{ + fprintf(stderr, + "usage: localize export OPTIONS CONFIGS...\n" + " REQUIRED OPTIONS\n" + " --settings SETTINGS The settings file to use. See CONFIGS below.\n" + " --root TREE_ROOT The location in Perforce of the files. e.g. //device\n" + " --target LOCALE The target locale. See LOCALES below.\n" + "\n" + " OPTIONAL OPTIONS\n" + " --out DIR Directory to put the output files. Defaults to the\n" + " current directory if not supplied. Files are\n" + " named as follows:\n" + " CONFIG-LOCALE-CURRENT_CL.xliff\n" + "\n" + "\n" + "usage: localize import XLIFF_FILE...\n" + "\n" + "Import a translated XLIFF file back into the tree.\n" + "\n" + "\n" + "usage: localize xlb XMB_FILE VALUES_FILES...\n" + "\n" + "Read resource files from the tree file and write the corresponding XLB file\n" + "\n" + "Supply all of the android resource files (values files) to export after that.\n" + "\n" + "\n" + "\n" + "CONFIGS\n" + "\n" + "LOCALES\n" + "Locales are specified in the form en_US They will be processed correctly\n" + "to locate the resouce files in the tree.\n" + "\n" + "\n" + "usage: localize pseudo OPTIONS INFILE [OUTFILE]\n" + " OPTIONAL OPTIONS\n" + " --big Pad strings so they get longer.\n" + "\n" + "Read INFILE, an XLIFF file, and output a pseudotranslated version of that file. If\n" + "OUTFILE is specified, the results are written there; otherwise, the results are\n" + "written back to INFILE.\n" + "\n" + "\n" + "usage: localize rescheck FILES...\n" + "\n" + "Reads the base strings and prints warnings about bad resources from the given files.\n" + "\n"); + return 1; +} + +int +main(int argc, const char** argv) +{ + //open_log_file("log.txt"); + //g_logFile = stdout; + + if (argc == 2 && 0 == strcmp(argv[1], "--test")) { + return test(); + } + + if (argc < 2) { + return usage(); + } + + int index = 1; + + if (0 == strcmp("export", argv[index])) { + string settingsFile; + string rootDir; + string outDir; + string baseLocale = "en"; + string targetLocale; + string language, region; + vector<string> configs; + + index++; + while (index < argc) { + if (0 == strcmp("--settings", argv[index])) { + settingsFile = argv[index+1]; + index += 2; + } + else if (0 == strcmp("--root", argv[index])) { + rootDir = argv[index+1]; + index += 2; + } + else if (0 == strcmp("--out", argv[index])) { + outDir = argv[index+1]; + index += 2; + } + else if (0 == strcmp("--target", argv[index])) { + targetLocale = argv[index+1]; + index += 2; + } + else if (argv[index][0] == '-') { + fprintf(stderr, "unknown argument %s\n", argv[index]); + return usage(); + } + else { + break; + } + } + for (; index<argc; index++) { + configs.push_back(argv[index]); + } + + if (settingsFile == "" || rootDir == "" || configs.size() == 0 || targetLocale == "") { + return usage(); + } + if (!split_locale(targetLocale, &language, ®ion)) { + fprintf(stderr, "illegal --target locale: '%s'\n", targetLocale.c_str()); + return usage(); + } + + + return do_export(settingsFile, rootDir, outDir, targetLocale, configs); + } + else if (0 == strcmp("import", argv[index])) { + vector<string> xliffFilenames; + + index++; + for (; index<argc; index++) { + xliffFilenames.push_back(argv[index]); + } + + return do_merge(xliffFilenames); + } + else if (0 == strcmp("xlb", argv[index])) { + string outfile; + vector<string> resFiles; + + index++; + if (argc < index+1) { + return usage(); + } + + outfile = argv[index]; + + index++; + for (; index<argc; index++) { + resFiles.push_back(argv[index]); + } + + return do_xlb_export(outfile, resFiles); + } + else if (0 == strcmp("pseudo", argv[index])) { + string infile; + string outfile; + bool big = false; + + index++; + while (index < argc) { + if (0 == strcmp("--big", argv[index])) { + big = true; + index += 1; + } + else if (argv[index][0] == '-') { + fprintf(stderr, "unknown argument %s\n", argv[index]); + return usage(); + } + else { + break; + } + } + + if (index == argc-1) { + infile = argv[index]; + outfile = argv[index]; + } + else if (index == argc-2) { + infile = argv[index]; + outfile = argv[index+1]; + } + else { + fprintf(stderr, "unknown argument %s\n", argv[index]); + return usage(); + } + + return do_pseudo(infile, outfile, big); + } + else if (0 == strcmp("rescheck", argv[index])) { + vector<string> files; + + index++; + while (index < argc) { + if (argv[index][0] == '-') { + fprintf(stderr, "unknown argument %s\n", argv[index]); + return usage(); + } + else { + break; + } + } + for (; index<argc; index++) { + files.push_back(argv[index]); + } + + if (files.size() == 0) { + return usage(); + } + + return do_rescheck(files); + } + else { + return usage(); + } + + if (SourcePos::HasErrors()) { + SourcePos::PrintErrors(stderr); + return 1; + } + + return 0; +} + diff --git a/tools/localize/localize.h b/tools/localize/localize.h new file mode 100644 index 0000000..615d14e --- /dev/null +++ b/tools/localize/localize.h @@ -0,0 +1,40 @@ +#ifndef LOCALIZE_H +#define LOCALIZE_H + +#include "XLIFFFile.h" + +#include <map> +#include <string> + +using namespace std; + +struct Reject +{ + string file; + string name; + string comment; +}; + +struct Settings +{ + string id; + string oldVersion; + string currentVersion; + vector<string> apps; + vector<Reject> reject; +}; + +int read_settings(const string& filename, map<string,Settings>* result, const string& rootDir); +string translated_file_name(const string& file, const string& locale); +bool keep_this_trans_unit(const string& file, const TransUnit& unit, void* cookie); +int validate_config(const string& settingsFile, const map<string,Settings>& settings, + const string& configs); +int validate_configs(const string& settingsFile, const map<string,Settings>& settings, + const vector<string>& configs); +int select_files(vector<string> *resFiles, const string& config, + const map<string,Settings>& settings, const string& rootDir); +int select_files(vector<vector<string> > *allResFiles, const vector<string>& configs, + const map<string,Settings>& settings, const string& rootDir); + + +#endif // LOCALIZE_H diff --git a/tools/localize/localize_test.cpp b/tools/localize/localize_test.cpp new file mode 100644 index 0000000..63d904c --- /dev/null +++ b/tools/localize/localize_test.cpp @@ -0,0 +1,219 @@ +#include "XLIFFFile.h" +#include "ValuesFile.h" +#include "localize.h" + +int pseudolocalize_xliff(XLIFFFile* xliff, bool expand); + +static int +test_filename(const string& file, const string& locale, const string& expected) +{ + string result = translated_file_name(file, locale); + if (result != expected) { + fprintf(stderr, "translated_file_name test failed\n"); + fprintf(stderr, " locale='%s'\n", locale.c_str()); + fprintf(stderr, " expected='%s'\n", expected.c_str()); + fprintf(stderr, " result='%s'\n", result.c_str()); + return 1; + } else { + if (false) { + fprintf(stderr, "translated_file_name test passed\n"); + fprintf(stderr, " locale='%s'\n", locale.c_str()); + fprintf(stderr, " expected='%s'\n", expected.c_str()); + fprintf(stderr, " result='%s'\n", result.c_str()); + } + return 0; + } +} + +static int +translated_file_name_test() +{ + bool all = true; + int err = 0; + + if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz_ZZ", + "//device/samples/NotePad/res/values-zz-rZZ/strings.xml"); + + if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "zz", + "//device/samples/NotePad/res/values-zz/strings.xml"); + + if (all) err |= test_filename("//device/samples/NotePad/res/values/strings.xml", "", + "//device/samples/NotePad/res/values/strings.xml"); + + return err; +} + +bool +return_false(const string&, const TransUnit& unit, void* cookie) +{ + return false; +} + +static int +delete_trans_units() +{ + XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff"); + if (xliff == NULL) { + printf("couldn't read file\n"); + return 1; + } + if (false) { + printf("XLIFF was [[%s]]\n", xliff->ToString().c_str()); + } + + xliff->Filter(return_false, NULL); + + if (false) { + printf("XLIFF is [[%s]]\n", xliff->ToString().c_str()); + + set<StringResource> const& strings = xliff->GetStringResources(); + printf("strings.size=%zd\n", strings.size()); + for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) { + const StringResource& str = *it; + printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(), + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.pos.ToString().c_str(), str.file.c_str(), str.version, + str.versionString.c_str()); + } + } + + return 0; +} + +static int +filter_trans_units() +{ + XLIFFFile* xliff = XLIFFFile::Parse("testdata/strip_xliff.xliff"); + if (xliff == NULL) { + printf("couldn't read file\n"); + return 1; + } + + if (false) { + printf("XLIFF was [[%s]]\n", xliff->ToString().c_str()); + } + + Settings setting; + xliff->Filter(keep_this_trans_unit, &setting); + + if (false) { + printf("XLIFF is [[%s]]\n", xliff->ToString().c_str()); + + set<StringResource> const& strings = xliff->GetStringResources(); + printf("strings.size=%zd\n", strings.size()); + for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) { + const StringResource& str = *it; + printf("STRING!!! id=%s value='%s' pos=%s file=%s version=%d(%s)\n", str.id.c_str(), + str.value->ContentsToString(ANDROID_NAMESPACES).c_str(), + str.pos.ToString().c_str(), str.file.c_str(), str.version, + str.versionString.c_str()); + } + } + + return 0; +} + +static int +settings_test() +{ + int err; + map<string,Settings> settings; + map<string,Settings>::iterator it; + + err = read_settings("testdata/config.xml", &settings, "//asdf"); + if (err != 0) { + return err; + } + + if (false) { + for (it=settings.begin(); it!=settings.end(); it++) { + const Settings& setting = it->second; + printf("CONFIG:\n"); + printf(" id='%s'\n", setting.id.c_str()); + printf(" oldVersion='%s'\n", setting.oldVersion.c_str()); + printf(" currentVersion='%s'\n", setting.currentVersion.c_str()); + int i=0; + for (vector<string>::const_iterator app=setting.apps.begin(); + app!=setting.apps.end(); app++) { + printf(" apps[%02d]='%s'\n", i, app->c_str()); + i++; + } + i=0; + for (vector<Reject>::const_iterator reject=setting.reject.begin(); + reject!=setting.reject.end(); reject++) { + i++; + printf(" reject[%02d]=('%s','%s','%s')\n", i, reject->file.c_str(), + reject->name.c_str(), reject->comment.c_str()); + } + } + } + + for (it=settings.begin(); it!=settings.end(); it++) { + const Settings& setting = it->second; + if (it->first != setting.id) { + fprintf(stderr, "it->first='%s' setting.id='%s'\n", it->first.c_str(), + setting.id.c_str()); + err |= 1; + } + } + + + return err; +} + +static int +test_one_pseudo(bool big, const char* expected) +{ + XLIFFFile* xliff = XLIFFFile::Parse("testdata/pseudo.xliff"); + if (xliff == NULL) { + printf("couldn't read file\n"); + return 1; + } + if (false) { + printf("XLIFF was [[%s]]\n", xliff->ToString().c_str()); + } + + pseudolocalize_xliff(xliff, big); + string newString = xliff->ToString(); + delete xliff; + + if (false) { + printf("XLIFF is [[%s]]\n", newString.c_str()); + } + + if (false && newString != expected) { + fprintf(stderr, "xliff didn't translate as expected\n"); + fprintf(stderr, "newString=[[%s]]\n", newString.c_str()); + fprintf(stderr, "expected=[[%s]]\n", expected); + return 1; + } + + return 0; +} + +static int +pseudolocalize_test() +{ + int err = 0; + + err |= test_one_pseudo(false, ""); + //err |= test_one_pseudo(true, ""); + + return err; +} + +int +localize_test() +{ + bool all = true; + int err = 0; + + if (all) err |= translated_file_name_test(); + if (all) err |= delete_trans_units(); + if (all) err |= filter_trans_units(); + if (all) err |= settings_test(); + if (all) err |= pseudolocalize_test(); + + return err; +} + diff --git a/tools/localize/log.h b/tools/localize/log.h new file mode 100644 index 0000000..4a5fa7f --- /dev/null +++ b/tools/localize/log.h @@ -0,0 +1,7 @@ +#ifndef LOG_H +#define LOG_H + +void log_printf(const char* fmt, ...); + +#endif // LOG_H + diff --git a/tools/localize/merge_res_and_xliff.cpp b/tools/localize/merge_res_and_xliff.cpp new file mode 100644 index 0000000..58a6554 --- /dev/null +++ b/tools/localize/merge_res_and_xliff.cpp @@ -0,0 +1,391 @@ +#include "merge_res_and_xliff.h" + +#include "file_utils.h" +#include "Perforce.h" +#include "log.h" + +static set<StringResource>::const_iterator +find_id(const set<StringResource>& s, const string& id, int index) +{ + for (set<StringResource>::const_iterator it = s.begin(); it != s.end(); it++) { + if (it->id == id && it->index == index) { + return it; + } + } + return s.end(); +} + +static set<StringResource>::const_iterator +find_in_xliff(const set<StringResource>& s, const string& filename, const string& id, int index, + int version, const Configuration& config) +{ + for (set<StringResource>::const_iterator it = s.begin(); it != s.end(); it++) { + if (it->file == filename && it->id == id && it->index == index && it->version == version + && it->config == config) { + return it; + } + } + return s.end(); +} + + +static void +printit(const set<StringResource>& s, const set<StringResource>::const_iterator& it) +{ + if (it == s.end()) { + printf("(none)\n"); + } else { + printf("id=%s index=%d config=%s file=%s value='%s'\n", it->id.c_str(), it->index, + it->config.ToString().c_str(), it->file.c_str(), + it->value->ToString(ANDROID_NAMESPACES).c_str()); + } +} + +StringResource +convert_resource(const StringResource& s, const string& file, const Configuration& config, + int version, const string& versionString) +{ + return StringResource(s.pos, file, config, s.id, s.index, s.value ? s.value->Clone() : NULL, + version, versionString, s.comment); +} + +static bool +resource_has_contents(const StringResource& res) +{ + XMLNode* value = res.value; + if (value == NULL) { + return false; + } + string contents = value->ContentsToString(ANDROID_NAMESPACES); + return contents != ""; +} + +ValuesFile* +merge_res_and_xliff(const ValuesFile* en_currentFile, + const ValuesFile* xx_currentFile, const ValuesFile* xx_oldFile, + const string& filename, const XLIFFFile* xliffFile) +{ + bool success = true; + + Configuration en_config = xliffFile->SourceConfig(); + Configuration xx_config = xliffFile->TargetConfig(); + string currentVersion = xliffFile->CurrentVersion(); + + ValuesFile* result = new ValuesFile(xx_config); + + set<StringResource> en_cur = en_currentFile->GetStrings(); + set<StringResource> xx_cur = xx_currentFile->GetStrings(); + set<StringResource> xx_old = xx_oldFile->GetStrings(); + set<StringResource> xliff = xliffFile->GetStringResources(); + + // for each string in en_current + for (set<StringResource>::const_iterator en_c = en_cur.begin(); + en_c != en_cur.end(); en_c++) { + set<StringResource>::const_iterator xx_c = find_id(xx_cur, en_c->id, en_c->index); + set<StringResource>::const_iterator xx_o = find_id(xx_old, en_c->id, en_c->index); + set<StringResource>::const_iterator xlf = find_in_xliff(xliff, en_c->file, en_c->id, + en_c->index, CURRENT_VERSION, xx_config); + + if (false) { + printf("\nen_c: "); printit(en_cur, en_c); + printf("xx_c: "); printit(xx_cur, xx_c); + printf("xx_o: "); printit(xx_old, xx_o); + printf("xlf: "); printit(xliff, xlf); + } + + // if it changed between xx_old and xx_current, use xx_current + // (someone changed it by hand) + if (xx_o != xx_old.end() && xx_c != xx_cur.end()) { + string xx_o_value = xx_o->value->ToString(ANDROID_NAMESPACES); + string xx_c_value = xx_c->value->ToString(ANDROID_NAMESPACES); + if (xx_o_value != xx_c_value && xx_c_value != "") { + StringResource r(convert_resource(*xx_c, filename, xx_config, + CURRENT_VERSION, currentVersion)); + if (resource_has_contents(r)) { + result->AddString(r); + } + continue; + } + } + + // if it is present in xliff, use that + // (it just got translated) + if (xlf != xliff.end() && xlf->value->ToString(ANDROID_NAMESPACES) != "") { + StringResource r(convert_resource(*xlf, filename, xx_config, + CURRENT_VERSION, currentVersion)); + if (resource_has_contents(r)) { + result->AddString(r); + } + } + + // if it is present in xx_current, use that + // (it was already translated, and not retranslated) + // don't filter out empty strings if they were added by hand, the above code just + // guarantees that this tool never adds an empty one. + if (xx_c != xx_cur.end()) { + StringResource r(convert_resource(*xx_c, filename, xx_config, + CURRENT_VERSION, currentVersion)); + result->AddString(r); + } + + // othwerwise, leave it out. The resource fall-through code will use the English + // one at runtime, and the xliff export code will pick it up for translation next time. + } + + if (success) { + return result; + } else { + delete result; + return NULL; + } +} + + +struct MergedFile { + XLIFFFile* xliff; + string xliffFilename; + string original; + string translated; + ValuesFile* en_current; + ValuesFile* xx_current; + ValuesFile* xx_old; + ValuesFile* xx_new; + string xx_new_text; + string xx_new_filename; + bool new_file; + bool deleted_file; + + MergedFile(); + MergedFile(const MergedFile&); +}; + +struct compare_filenames { + bool operator()(const MergedFile& lhs, const MergedFile& rhs) const + { + return lhs.original < rhs.original; + } +}; + +MergedFile::MergedFile() + :xliff(NULL), + xliffFilename(), + original(), + translated(), + en_current(NULL), + xx_current(NULL), + xx_old(NULL), + xx_new(NULL), + xx_new_text(), + xx_new_filename(), + new_file(false), + deleted_file(false) +{ +} + +MergedFile::MergedFile(const MergedFile& that) + :xliff(that.xliff), + xliffFilename(that.xliffFilename), + original(that.original), + translated(that.translated), + en_current(that.en_current), + xx_current(that.xx_current), + xx_old(that.xx_old), + xx_new(that.xx_new), + xx_new_text(that.xx_new_text), + xx_new_filename(that.xx_new_filename), + new_file(that.new_file), + deleted_file(that.deleted_file) +{ +} + + +typedef set<MergedFile, compare_filenames> MergedFileSet; + +int +do_merge(const vector<string>& xliffFilenames) +{ + int err = 0; + MergedFileSet files; + + printf("\rPreparing..."); fflush(stdout); + string currentChange = Perforce::GetCurrentChange(true); + + // for each xliff, make a MergedFile record and do a little error checking + for (vector<string>::const_iterator xliffFilename=xliffFilenames.begin(); + xliffFilename!=xliffFilenames.end(); xliffFilename++) { + XLIFFFile* xliff = XLIFFFile::Parse(*xliffFilename); + if (xliff == NULL) { + fprintf(stderr, "localize import: unable to read file %s\n", xliffFilename->c_str()); + err = 1; + continue; + } + + set<string> xf = xliff->Files(); + for (set<string>::const_iterator f=xf.begin(); f!=xf.end(); f++) { + MergedFile mf; + mf.xliff = xliff; + mf.xliffFilename = *xliffFilename; + mf.original = *f; + mf.translated = translated_file_name(mf.original, xliff->TargetConfig().locale); + log_printf("mf.translated=%s mf.original=%s locale=%s\n", mf.translated.c_str(), + mf.original.c_str(), xliff->TargetConfig().locale.c_str()); + + if (files.find(mf) != files.end()) { + fprintf(stderr, "%s: duplicate string resources for file %s\n", + xliffFilename->c_str(), f->c_str()); + fprintf(stderr, "%s: previously defined here.\n", + files.find(mf)->xliffFilename.c_str()); + err = 1; + continue; + } + files.insert(mf); + } + } + + size_t deletedFileCount = 0; + size_t J = files.size() * 3; + size_t j = 1; + // Read all of the files from perforce. + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast<MergedFile*>(&(*mf)); + // file->en_current + print_file_status(j++, J); + file->en_current = get_values_file(file->original, file->xliff->SourceConfig(), + CURRENT_VERSION, currentChange, true); + if (file->en_current == NULL) { + // deleted file + file->deleted_file = true; + deletedFileCount++; + continue; + } + + // file->xx_current; + print_file_status(j++, J); + file->xx_current = get_values_file(file->translated, file->xliff->TargetConfig(), + CURRENT_VERSION, currentChange, false); + if (file->xx_current == NULL) { + file->xx_current = new ValuesFile(file->xliff->TargetConfig()); + file->new_file = true; + } + + // file->xx_old (note that the xliff's current version is our old version, because that + // was the current version when it was exported) + print_file_status(j++, J); + file->xx_old = get_values_file(file->translated, file->xliff->TargetConfig(), + OLD_VERSION, file->xliff->CurrentVersion(), false); + if (file->xx_old == NULL) { + file->xx_old = new ValuesFile(file->xliff->TargetConfig()); + file->new_file = true; + } + } + + // merge them + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast<MergedFile*>(&(*mf)); + if (file->deleted_file) { + continue; + } + file->xx_new = merge_res_and_xliff(file->en_current, file->xx_current, file->xx_old, + file->original, file->xliff); + } + + // now is a good time to stop if there was an error + if (err != 0) { + return err; + } + + // locate the files + j = 1; + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast<MergedFile*>(&(*mf)); + print_file_status(j++, J, "Locating"); + + file->xx_new_filename = Perforce::Where(file->translated, true); + if (file->xx_new_filename == "") { + fprintf(stderr, "\nWas not able to determine the location of depot file %s\n", + file->translated.c_str()); + err = 1; + } + } + + if (err != 0) { + return err; + } + + // p4 edit the files + // only do this if it changed - no need to submit files that haven't changed meaningfully + vector<string> filesToEdit; + vector<string> filesToAdd; + vector<string> filesToDelete; + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast<MergedFile*>(&(*mf)); + if (file->deleted_file) { + filesToDelete.push_back(file->xx_new_filename); + continue; + } + string xx_current_text = file->xx_current->ToString(); + string xx_new_text = file->xx_new->ToString(); + if (xx_current_text != xx_new_text) { + if (file->xx_new->GetStrings().size() == 0) { + file->deleted_file = true; + filesToDelete.push_back(file->xx_new_filename); + } else { + file->xx_new_text = xx_new_text; + if (file->new_file) { + filesToAdd.push_back(file->xx_new_filename); + } else { + filesToEdit.push_back(file->xx_new_filename); + } + } + } + } + if (filesToAdd.size() == 0 && filesToEdit.size() == 0 && deletedFileCount == 0) { + printf("\nAll of the files are the same. Nothing to change.\n"); + return 0; + } + if (filesToEdit.size() > 0) { + printf("\np4 editing files...\n"); + if (0 != Perforce::EditFiles(filesToEdit, true)) { + return 1; + } + } + + + printf("\n"); + + for (MergedFileSet::iterator mf = files.begin(); mf != files.end(); mf++) { + MergedFile* file = const_cast<MergedFile*>(&(*mf)); + if (file->deleted_file) { + continue; + } + if (file->xx_new_text != "" && file->xx_new_filename != "") { + if (0 != write_to_file(file->xx_new_filename, file->xx_new_text)) { + err = 1; + } + } + } + + if (err != 0) { + return err; + } + + if (filesToAdd.size() > 0) { + printf("p4 adding %zd new files...\n", filesToAdd.size()); + err = Perforce::AddFiles(filesToAdd, true); + } + + if (filesToDelete.size() > 0) { + printf("p4 deleting %zd removed files...\n", filesToDelete.size()); + err = Perforce::DeleteFiles(filesToDelete, true); + } + + if (err != 0) { + return err; + } + + printf("\n" + "Theoretically, this merge was successfull. Next you should\n" + "review the diffs, get a code review, and submit it. Enjoy.\n\n"); + return 0; +} + diff --git a/tools/localize/merge_res_and_xliff.h b/tools/localize/merge_res_and_xliff.h new file mode 100644 index 0000000..acf2fff --- /dev/null +++ b/tools/localize/merge_res_and_xliff.h @@ -0,0 +1,13 @@ +#ifndef MERGE_RES_AND_XLIFF_H +#define MERGE_RES_AND_XLIFF_H + +#include "ValuesFile.h" +#include "XLIFFFile.h" + +ValuesFile* merge_res_and_xliff(const ValuesFile* en_current, + const ValuesFile* xx_current, const ValuesFile* xx_old, + const string& filename, const XLIFFFile* xliff); + +int do_merge(const vector<string>& xliffFilenames); + +#endif // MERGE_RES_AND_XLIFF_H diff --git a/tools/localize/merge_res_and_xliff_test.cpp b/tools/localize/merge_res_and_xliff_test.cpp new file mode 100644 index 0000000..5a2b0f4 --- /dev/null +++ b/tools/localize/merge_res_and_xliff_test.cpp @@ -0,0 +1,47 @@ +#include "merge_res_and_xliff.h" + + +int +merge_test() +{ + Configuration english; + english.locale = "en_US"; + Configuration translated; + translated.locale = "zz_ZZ"; + + ValuesFile* en_current = ValuesFile::ParseFile("testdata/merge_en_current.xml", english, + CURRENT_VERSION, "3"); + if (en_current == NULL) { + fprintf(stderr, "merge_test: unable to read testdata/merge_en_current.xml\n"); + return 1; + } + + ValuesFile* xx_current = ValuesFile::ParseFile("testdata/merge_xx_current.xml", translated, + CURRENT_VERSION, "3"); + if (xx_current == NULL) { + fprintf(stderr, "merge_test: unable to read testdata/merge_xx_current.xml\n"); + return 1; + } + ValuesFile* xx_old = ValuesFile::ParseFile("testdata/merge_xx_old.xml", translated, + OLD_VERSION, "2"); + if (xx_old == NULL) { + fprintf(stderr, "merge_test: unable to read testdata/merge_xx_old.xml\n"); + return 1; + } + + XLIFFFile* xliff = XLIFFFile::Parse("testdata/merge.xliff"); + + ValuesFile* result = merge_res_and_xliff(en_current, xx_current, xx_old, + "//device/tools/localize/testdata/res/values/strings.xml", xliff); + + if (result == NULL) { + fprintf(stderr, "merge_test: result is NULL\n"); + return 1; + } + + printf("======= RESULT =======\n%s===============\n", result->ToString().c_str()); + + return 0; +} + + diff --git a/tools/localize/res_check.cpp b/tools/localize/res_check.cpp new file mode 100644 index 0000000..0fab98a --- /dev/null +++ b/tools/localize/res_check.cpp @@ -0,0 +1,106 @@ +#include "res_check.h" +#include "localize.h" +#include "file_utils.h" +#include "ValuesFile.h" + +#include <stdio.h> + +static int check_file(const ValuesFile* file); +static int check_value(const SourcePos& pos, const XMLNode* value); +static int scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth = 0); + +int +do_rescheck(const vector<string>& files) +{ + int err; + + Configuration english; + english.locale = "en_US"; + + for (size_t i=0; i<files.size(); i++) { + const string filename = files[i]; + ValuesFile* valuesFile = get_local_values_file(filename, english, CURRENT_VERSION, + "0", true); + if (valuesFile != NULL) { + err |= check_file(valuesFile); + delete valuesFile; + } else { + err |= 1; + } + } + + return err; +} + +static int +check_file(const ValuesFile* file) +{ + int err = 0; + set<StringResource> strings = file->GetStrings(); + for (set<StringResource>::iterator it=strings.begin(); it!=strings.end(); it++) { + XMLNode* value = it->value; + if (value != NULL) { + err |= check_value(it->pos, value); + } + } + return err; +} + +static bool +contains_percent(const string& str) +{ + const size_t len = str.length(); + for (size_t i=0; i<len; i++) { + char c = str[i]; + if (c == '%') { + return true; + } + } + return false; +} + +static int +check_value(const SourcePos& pos, const XMLNode* value) +{ + int err = 0; + err |= scan_for_unguarded_format(pos, value); + return err; +} + +static bool +is_xliff_block(const string& ns, const string& name) +{ + if (ns == XLIFF_XMLNS) { + return name == "g"; + } else { + return false; + } +} + +static int +scan_for_unguarded_format(const SourcePos& pos, const string& string) +{ + bool containsPercent = contains_percent(string); + if (containsPercent) { + pos.Error("unguarded percent: '%s'\n", string.c_str()); + } + return 0; +} + +static int +scan_for_unguarded_format(const SourcePos& pos, const XMLNode* value, int depth) +{ + if (value->Type() == XMLNode::ELEMENT) { + int err = 0; + if (depth == 0 || !is_xliff_block(value->Namespace(), value->Name())) { + const vector<XMLNode*>& children = value->Children(); + for (size_t i=0; i<children.size(); i++) { + err |= scan_for_unguarded_format(pos, children[i], depth+1); + } + } + return err; + } else { + return scan_for_unguarded_format(pos, value->Text()); + } +} + diff --git a/tools/localize/res_check.h b/tools/localize/res_check.h new file mode 100644 index 0000000..86e7ce6 --- /dev/null +++ b/tools/localize/res_check.h @@ -0,0 +1,12 @@ +#ifndef RESCHECK_H +#define RESCHECK_H + +#include <map> +#include <string> +#include <vector> + +using namespace std; + +int do_rescheck(const vector<string>& files); + +#endif // RESCHECK_H diff --git a/tools/localize/test.cpp b/tools/localize/test.cpp new file mode 100644 index 0000000..5fa2c17 --- /dev/null +++ b/tools/localize/test.cpp @@ -0,0 +1,31 @@ +#include "SourcePos.h" +#include <stdio.h> + +int ValuesFile_test(); +int XLIFFFile_test(); +int XMLHandler_test(); +int Perforce_test(); +int localize_test(); +int merge_test(); + +int +test() +{ + bool all = true; + int err = 0; + + if (all) err |= XMLHandler_test(); + if (all) err |= ValuesFile_test(); + if (all) err |= XLIFFFile_test(); + if (all) err |= Perforce_test(); + if (all) err |= localize_test(); + if (all) err |= merge_test(); + + if (err != 0) { + fprintf(stderr, "some tests failed\n"); + } else { + fprintf(stderr, "all tests passed\n"); + } + + return err; +} diff --git a/tools/localize/testdata/config.xml b/tools/localize/testdata/config.xml new file mode 100644 index 0000000..affa140 --- /dev/null +++ b/tools/localize/testdata/config.xml @@ -0,0 +1,15 @@ +<localize-config> + <configuration id="system" + old-cl="1" + new-cl="43019"> + <app dir="apps/common" /> + </configuration> + <configuration id="samples" + old-cl="24801" + new-cl="43019"> + <app dir="samples/NotePad" /> + <reject file="samples/NotePad/res/values/strings.xml" name="string:menu_delete"> + QA says this sounds <b>rude</b>. + </reject> + </configuration> +</localize-config> diff --git a/tools/localize/testdata/import.xliff b/tools/localize/testdata/import.xliff new file mode 100644 index 0000000..b99b739 --- /dev/null +++ b/tools/localize/testdata/import.xliff @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" + version="1.2" + > + <file datatype="x-android-res" + original="//device/tools/localize/testdata/res/values/strings.xml" + product-version="1.0" + date="08:10:54 12/07/07 PST" + source-language="en_US" + product-name="kila" + target-language="zz_ZZ" + build-num="44391" + > + <body> + <trans-unit id="string:changed_in_xx"> + <source>aaa</source> + <target>AAA</target> + </trans-unit> + <trans-unit id="string:first_translation"> + <source>bbb</source> + <target>BBB</target> + </trans-unit> + <trans-unit id="string:deleted_string"> + <source>ddd</source> + <target>DDDD</target> + </trans-unit> + <trans-unit id="array:0:growing_array"> + <source>1-One</source> + <target>1-oNE</target> + </trans-unit> + <trans-unit id="array:1:growing_array"> + <source>1-Two</source> + <target>1-tWO</target> + </trans-unit> + <trans-unit id="array:2:growing_array"> + <source>1-Three</source> + <target>1-tHREE</target> + </trans-unit> + <trans-unit id="array:0:shrinking_array"> + <source>2-One</source> + <target>2-oNE</target> + </trans-unit> + <trans-unit id="array:1:shrinking_array"> + <source>2-Two</source> + <target>2-tWO</target> + </trans-unit> + <trans-unit id="array:2:shrinking_array"> + <source>2-Three</source> + <target>2-tHREE</target> + </trans-unit> + <trans-unit id="array:3:shrinking_array"> + <source>2-Four</source> + <target>2-fOUR</target> + </trans-unit> + <trans-unit id="array:0:deleted_array"> + <source>4-One</source> + <target>4-oNE</target> + </trans-unit> + <trans-unit id="array:1:deleted_array"> + <source>4-Two</source> + <target>4-tWO</target> + </trans-unit> + <trans-unit id="array:2:deleted_array"> + <source>4-Three</source> + <target>4-tHREE</target> + </trans-unit> + + </body> + </file> +</xliff> + + diff --git a/tools/localize/testdata/merge.xliff b/tools/localize/testdata/merge.xliff new file mode 100644 index 0000000..2b78c45 --- /dev/null +++ b/tools/localize/testdata/merge.xliff @@ -0,0 +1,72 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" + version="1.2" + > + <file datatype="x-android-res" + original="testdata/merge_en_current.xml" + product-version="1.0" + date="08:10:54 12/07/07 PST" + source-language="en-US" + product-name="kila" + target-language="zz-ZZ" + build-num="44391" + > + <body> + <trans-unit id="string:changed_in_xx"> + <source>aaa</source> + <target>AAA</target> + </trans-unit> + <trans-unit id="string:first_translation"> + <source>bbb</source> + <target>BBB</target> + </trans-unit> + <trans-unit id="string:deleted_string"> + <source>ddd</source> + <target>DDDD</target> + </trans-unit> + <trans-unit id="array:0:growing_array"> + <source>1-One</source> + <target>1-oNE</target> + </trans-unit> + <trans-unit id="array:1:growing_array"> + <source>1-Two</source> + <target>1-tWO</target> + </trans-unit> + <trans-unit id="array:2:growing_array"> + <source>1-Three</source> + <target>1-tHREE</target> + </trans-unit> + <trans-unit id="array:0:shrinking_array"> + <source>2-One</source> + <target>2-oNE</target> + </trans-unit> + <trans-unit id="array:1:shrinking_array"> + <source>2-Two</source> + <target>2-tWO</target> + </trans-unit> + <trans-unit id="array:2:shrinking_array"> + <source>2-Three</source> + <target>2-tHREE</target> + </trans-unit> + <trans-unit id="array:3:shrinking_array"> + <source>2-Four</source> + <target>2-fOUR</target> + </trans-unit> + <trans-unit id="array:0:deleted_array"> + <source>4-One</source> + <target>4-oNE</target> + </trans-unit> + <trans-unit id="array:1:deleted_array"> + <source>4-Two</source> + <target>4-tWO</target> + </trans-unit> + <trans-unit id="array:2:deleted_array"> + <source>4-Three</source> + <target>4-tHREE</target> + </trans-unit> + + </body> + </file> +</xliff> + + diff --git a/tools/localize/testdata/merge_en_current.xml b/tools/localize/testdata/merge_en_current.xml new file mode 100644 index 0000000..6a11e68 --- /dev/null +++ b/tools/localize/testdata/merge_en_current.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="changed_in_xx">aaa</string> + <string name="first_translation">bbb</string> + <string name="previously_translated">ccc</string> + <string name="new_string">ccc</string> + + <string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string> + + <array name="growing_array"> + <!-- somebody wrote a comment! --> + <item>1-One</item> + <item>1-Two</item> + <item>1-Three</item> + <item>1-Four</item> + </array> + <array name="shrinking_array"> + <!-- somebody wrote a comment! --> + <item>2-One</item> + <item>2-Two</item> + <item>2-Three</item> + </array> + <array name="new_array"> + <!-- somebody wrote a comment! --> + <item>3-One</item> + <item>3-Two</item> + <item>3-Three</item> + </array> +</resources> diff --git a/tools/localize/testdata/merge_en_old.xml b/tools/localize/testdata/merge_en_old.xml new file mode 100644 index 0000000..933f98e --- /dev/null +++ b/tools/localize/testdata/merge_en_old.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="changed_in_xx">aaa</string> + <string name="first_translation">bbb</string> + <string name="previously_translated">ccc</string> + <string name="deleted_string">ddd</string> + + <string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string> + + <array name="growing_array"> + <!-- somebody wrote a comment! --> + <item>1-One</item> + <item>1-Two</item> + <item>1-Three</item> + </array> + <array name="shrinking_array"> + <!-- somebody wrote a comment! --> + <item>2-One</item> + <item>2-Two</item> + <item>2-Three</item> + <item>2-Four</item> + </array> + <array name="deleted_array"> + <!-- somebody wrote a comment! --> + <item>4-One</item> + <item>4-Two</item> + <item>4-Three</item> + </array> +</resources> + diff --git a/tools/localize/testdata/merge_xx_current.xml b/tools/localize/testdata/merge_xx_current.xml new file mode 100644 index 0000000..c2a783d --- /dev/null +++ b/tools/localize/testdata/merge_xx_current.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="changed_in_xx">AAAA</string> + <string name="previously_translated">CCC</string> +</resources> + + diff --git a/tools/localize/testdata/merge_xx_old.xml b/tools/localize/testdata/merge_xx_old.xml new file mode 100644 index 0000000..9d3a7d8 --- /dev/null +++ b/tools/localize/testdata/merge_xx_old.xml @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="changed_in_xx">aaa</string> + <string name="previously_translated">CCC</string> +</resources> + diff --git a/tools/localize/testdata/pseudo.xliff b/tools/localize/testdata/pseudo.xliff new file mode 100644 index 0000000..5b44f86 --- /dev/null +++ b/tools/localize/testdata/pseudo.xliff @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" + version="1.2" + > + <file datatype="x-android-res" + original="//device/tools/localization/tests/res/values/strings.xml" + product-version="1.0" + date="08:10:54 12/07/07 PST" + source-language="en-US" + product-name="kila" + target-language="zz-ZZ" + build-num="32138" + > + <body> + <trans-unit id="string:complex"> + <source>First <g id="string:complex:0" ctype="underline">underline</g>, <g id="string:complex:1" ctype="italic">italic<g id="string:complex:2" ctype="bold">italicbold</g></g> End </source> + </trans-unit> + <trans-unit id="string:complex-quoted"> + <source xml:space="preserve">First <g id="string:complex-quoted:0" ctype="underline">underline</g>, <g id="string:complex-quoted:1" ctype="italic">italic<g id="string:complex-quoted:2" ctype="bold">italicbold</g></g> End</source> + </trans-unit> + <trans-unit id="string:simple"> + <source>Simple</source> + </trans-unit> + <trans-unit id="array:0:simple"> + <source>Simple</source> + </trans-unit> + <trans-unit id="array:1:simple"> + <source>Simple</source> + </trans-unit> + <trans-unit id="string:simple-quoted"> + <source xml:space="preserve"> Quote</source> + <alt-trans> + <source xml:lang="en" xml:space="preserve"> OLD Quote</source> + <target xml:lang="xx"> OLD Ờũỡŧę</target> + </alt-trans> + </trans-unit> + </body> + </file> +</xliff> + diff --git a/tools/localize/testdata/res/values-zz-rZZ/strings.xml b/tools/localize/testdata/res/values-zz-rZZ/strings.xml new file mode 100644 index 0000000..c2a783d --- /dev/null +++ b/tools/localize/testdata/res/values-zz-rZZ/strings.xml @@ -0,0 +1,22 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="changed_in_xx">AAAA</string> + <string name="previously_translated">CCC</string> +</resources> + + diff --git a/tools/localize/testdata/res/values/strings.xml b/tools/localize/testdata/res/values/strings.xml new file mode 100644 index 0000000..6a11e68 --- /dev/null +++ b/tools/localize/testdata/res/values/strings.xml @@ -0,0 +1,44 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="changed_in_xx">aaa</string> + <string name="first_translation">bbb</string> + <string name="previously_translated">ccc</string> + <string name="new_string">ccc</string> + + <string name="formatted_string"><b>bold</b><i>italic<u>italic_underline</u></i><u>underline</u></string> + + <array name="growing_array"> + <!-- somebody wrote a comment! --> + <item>1-One</item> + <item>1-Two</item> + <item>1-Three</item> + <item>1-Four</item> + </array> + <array name="shrinking_array"> + <!-- somebody wrote a comment! --> + <item>2-One</item> + <item>2-Two</item> + <item>2-Three</item> + </array> + <array name="new_array"> + <!-- somebody wrote a comment! --> + <item>3-One</item> + <item>3-Two</item> + <item>3-Three</item> + </array> +</resources> diff --git a/tools/localize/testdata/strip_xliff.xliff b/tools/localize/testdata/strip_xliff.xliff new file mode 100644 index 0000000..9254cf2 --- /dev/null +++ b/tools/localize/testdata/strip_xliff.xliff @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" + version="1.2" + > + <file datatype="x-android-res" + original="//device/tools/localization/tests/res/values/strings.xml" + product-version="1.0" + date="08:10:54 12/07/07 PST" + source-language="en-US" + product-name="kila" + target-language="zz-ZZ" + build-num="32138" + > + <body> + + <trans-unit id="string:string-000-0"> + </trans-unit> + <trans-unit id="string:string-001-0"> + <alt-trans> + <source xml:lang="en" xml:space="preserve">source</source> + </alt-trans> + </trans-unit> + <trans-unit id="string:string-010-0"> + <alt-trans> + <target xml:lang="zz" xml:space="preserve">target</target> + </alt-trans> + </trans-unit> + <trans-unit id="string:string-011-0"> + <alt-trans> + <source xml:lang="en" xml:space="preserve">source</source> + <target xml:lang="zz" xml:space="preserve">target</target> + </alt-trans> + </trans-unit> + + <trans-unit id="string:string-100-1"> + <source xml:space="preserve">source</source> + </trans-unit> + <trans-unit id="string:string-101-1"> + <source xml:space="preserve">source</source> + <alt-trans> + <source xml:lang="en" xml:space="preserve">source</source> + </alt-trans> + </trans-unit> + <trans-unit id="string:string-110-1"> + <source xml:space="preserve">source</source> + <alt-trans> + <target xml:lang="zz" xml:space="preserve">target</target> + </alt-trans> + </trans-unit> + + <trans-unit id="string:string-111-0"> + <source xml:space="preserve">source</source> + <alt-trans> + <source xml:lang="en" xml:space="preserve">source</source> + <target xml:lang="zz" xml:space="preserve">target</target> + </alt-trans> + </trans-unit> + <trans-unit id="string:string-111-1"> + <source xml:space="preserve">source</source> + <alt-trans> + <source xml:lang="en" xml:space="preserve">alt-source</source> + <target xml:lang="zz" xml:space="preserve">target</target> + </alt-trans> + </trans-unit> + + </body> + </file> +</xliff> + + diff --git a/tools/localize/testdata/values/strings.xml b/tools/localize/testdata/values/strings.xml new file mode 100644 index 0000000..5e8d43d --- /dev/null +++ b/tools/localize/testdata/values/strings.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- Copyright (C) 2007 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<resources + xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"> + <string name="test1">Discard</string> + <!-- comment --> + <string name="test2">a<b>b<i>c</i></b>d</string> + <string name="test3">a<xliff:g a="b" xliff:a="asdf">bBb</xliff:g>C</string> + + <!-- Email address types from android.provider.Contacts --> + <array name="emailAddressTypes"> + <!-- somebody wrote a comment! --> + <item>Email</item> + <item>Home</item> + <item>Work</item> + <item>Other\u2026</item> + </array> +</resources> diff --git a/tools/localize/testdata/xliff1.xliff b/tools/localize/testdata/xliff1.xliff new file mode 100644 index 0000000..55a8d8e --- /dev/null +++ b/tools/localize/testdata/xliff1.xliff @@ -0,0 +1,46 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<xliff xmlns="urn:oasis:names:tc:xliff:document:1.2" + version="1.2" + > + <file datatype="x-android-res" + original="//device/tools/localization/tests/res/values/strings.xml" + product-version="1.0" + date="08:10:54 12/07/07 PST" + source-language="en-US" + product-name="kila" + target-language="zz-ZZ" + build-num="32138" + > + <body> + <trans-unit id="string:complex"> + <source>First <g id="string:complex:0" ctype="underline">underline</g>, <g id="string:complex:1" ctype="italic">italic<g id="string:complex:2" ctype="bold">italicbold</g></g> End </source> + <target>Ḟịṙṩŧ , Ḛŋḋ </target> + </trans-unit> + <trans-unit id="string:complex-quoted"> + <source xml:space="preserve">First <g id="string:complex-quoted:0" ctype="underline">underline</g>, <g id="string:complex-quoted:1" ctype="italic">italic<g id="string:complex-quoted:2" ctype="bold">italicbold</g></g> End</source> + <target>Ḟịṙṩŧ , Ḛŋḋ</target> + </trans-unit> + <trans-unit id="string:simple"> + <source>Simple</source> + <target>Ṩịṃṕļę</target> + </trans-unit> + <trans-unit id="array:0:simple"> + <source>Simple</source> + <target>Ṩịṃṕļę</target> + </trans-unit> + <trans-unit id="array:1:simple"> + <source>Simple</source> + <target>Ṩịṃṕļę</target> + </trans-unit> + <trans-unit id="string:simple-quoted"> + <source xml:space="preserve"> Quote</source> + <target> Ờũỡŧę</target> + <alt-trans> + <source xml:lang="en" xml:space="preserve"> OLD Quote</source> + <target xml:lang="xx"> OLD Ờũỡŧę</target> + </alt-trans> + </trans-unit> + </body> + </file> +</xliff> + diff --git a/tools/localize/testdata/xml.xml b/tools/localize/testdata/xml.xml new file mode 100644 index 0000000..ef930d0 --- /dev/null +++ b/tools/localize/testdata/xml.xml @@ -0,0 +1,16 @@ +<ASDF> + <a id="system" + old-cl="1" + new-cl="43019"> + <app dir="apps/common" /> + </a> + <a id="samples" + old-cl="1" + new-cl="43019">asdf + <app dir="samples/NotePad" /> + <app dir="samples/LunarLander" /> + <something>a<b>,</b>b </something> + <exact xml:space="preserve">a<b>,</b>b </exact> + </a> +</ASDF> + diff --git a/tools/localize/xmb.cpp b/tools/localize/xmb.cpp new file mode 100644 index 0000000..236705f --- /dev/null +++ b/tools/localize/xmb.cpp @@ -0,0 +1,181 @@ +#include "xmb.h" + +#include "file_utils.h" +#include "localize.h" +#include "ValuesFile.h" +#include "XMLHandler.h" +#include "XLIFFFile.h" + +#include <map> + +using namespace std; + +const char *const NS_MAP[] = { + "xml", XMLNS_XMLNS, + NULL, NULL +}; + +set<string> g_tags; + +static string +strip_newlines(const string& str) +{ + string res; + const size_t N = str.length(); + for (size_t i=0; i<N; i++) { + char c = str[i]; + if (c != '\n' && c != '\r') { + res += c; + } else { + res += ' '; + } + } + return res; +} + +static int +rename_id_attribute(XMLNode* node) +{ + vector<XMLAttribute>& attrs = node->EditAttributes(); + const size_t I = attrs.size(); + for (size_t i=0; i<I; i++) { + XMLAttribute attr = attrs[i]; + if (attr.name == "id") { + attr.name = "name"; + attrs.erase(attrs.begin()+i); + attrs.push_back(attr); + return 0; + } + } + return 1; +} + +static int +convert_xliff_to_ph(XMLNode* node, int* phID) +{ + int err = 0; + if (node->Type() == XMLNode::ELEMENT) { + if (node->Namespace() == XLIFF_XMLNS) { + g_tags.insert(node->Name()); + node->SetName("", "ph"); + + err = rename_id_attribute(node); + if (err != 0) { + char name[30]; + (*phID)++; + sprintf(name, "id-%d", *phID); + node->EditAttributes().push_back(XMLAttribute("", "name", name)); + err = 0; + } + } + vector<XMLNode*>& children = node->EditChildren(); + const size_t I = children.size(); + for (size_t i=0; i<I; i++) { + err |= convert_xliff_to_ph(children[i], phID); + } + } + return err; +} + +XMLNode* +resource_to_xmb_msg(const StringResource& res) +{ + // the msg element + vector<XMLAttribute> attrs; + string name = res.pos.file; + name += ":"; + name += res.TypedID(); + attrs.push_back(XMLAttribute("", "name", name)); + attrs.push_back(XMLAttribute("", "desc", strip_newlines(res.comment))); + attrs.push_back(XMLAttribute(XMLNS_XMLNS, "space", "preserve")); + XMLNode* msg = XMLNode::NewElement(res.pos, "", "msg", attrs, XMLNode::EXACT); + + // the contents are in xliff/html, convert it to xliff + int err = 0; + XMLNode* value = res.value; + string tag = value->Name(); + int phID = 0; + for (vector<XMLNode*>::const_iterator it=value->Children().begin(); + it!=value->Children().end(); it++) { + err |= convert_html_to_xliff(*it, tag, msg, &phID); + } + + if (err != 0) { + return NULL; + } + + // and then convert that to xmb + for (vector<XMLNode*>::iterator it=msg->EditChildren().begin(); + it!=msg->EditChildren().end(); it++) { + err |= convert_xliff_to_ph(*it, &phID); + } + + if (err == 0) { + return msg; + } else { + return NULL; + } +} + +int +do_xlb_export(const string& outfile, const vector<string>& resFiles) +{ + int err = 0; + + size_t totalFileCount = resFiles.size(); + + Configuration english; + english.locale = "en_US"; + + set<StringResource> allResources; + + const size_t J = resFiles.size(); + for (size_t j=0; j<J; j++) { + string resFile = resFiles[j]; + + ValuesFile* valuesFile = get_local_values_file(resFile, english, CURRENT_VERSION, "", true); + if (valuesFile != NULL) { + set<StringResource> resources = valuesFile->GetStrings(); + allResources.insert(resources.begin(), resources.end()); + } else { + fprintf(stderr, "error reading file %s\n", resFile.c_str()); + } + + delete valuesFile; + } + + // Construct the XLB xml + vector<XMLAttribute> attrs; + attrs.push_back(XMLAttribute("", "locale", "en")); + XMLNode* localizationbundle = XMLNode::NewElement(GENERATED_POS, "", "localizationbundle", + attrs, XMLNode::PRETTY); + + for (set<StringResource>::iterator it=allResources.begin(); it!=allResources.end(); it++) { + XMLNode* msg = resource_to_xmb_msg(*it); + if (msg) { + localizationbundle->EditChildren().push_back(msg); + } else { + err = 1; + } + } + +#if 0 + for (set<string>::iterator it=g_tags.begin(); it!=g_tags.end(); it++) { + printf("tag: %s\n", it->c_str()); + } + printf("err=%d\n", err); +#endif + if (err == 0) { + FILE* f = fopen(outfile.c_str(), "wb"); + if (f == NULL) { + fprintf(stderr, "can't open outputfile: %s\n", outfile.c_str()); + return 1; + } + fprintf(f, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"); + fprintf(f, "%s\n", localizationbundle->ToString(NS_MAP).c_str()); + fclose(f); + } + + return err; +} + diff --git a/tools/localize/xmb.h b/tools/localize/xmb.h new file mode 100644 index 0000000..96492b1 --- /dev/null +++ b/tools/localize/xmb.h @@ -0,0 +1,11 @@ +#ifndef XMB_H +#define XMB_H + +#include <string> +#include <vector> + +using namespace std; + +int do_xlb_export(const string& outFile, const vector<string>& resFiles); + +#endif // XMB_H |