diff --git a/src/aim1_mod_maker/aim1_mod_maker.h b/src/aim1_mod_maker/aim1_mod_maker.h index 4b24d22..f397940 100644 --- a/src/aim1_mod_maker/aim1_mod_maker.h +++ b/src/aim1_mod_maker/aim1_mod_maker.h @@ -85,7 +85,7 @@ struct mod_maker { } } void write() { - m.save(fn); + m.save(fn, codepage); written = true; } auto &operator[](this auto &&d, const std::string &s) { @@ -475,7 +475,7 @@ struct mod_maker { if (!aim2_available()) { throw std::runtime_error{"aim2 is not available, setup it first"}; } - static auto m2 = db2{aim2_game_dir / "data" / "db", 1251}.open().to_map(); + static auto m2 = db2{aim2_game_dir / "data" / "db"}.open().to_map(1251); return m2; } @@ -489,7 +489,7 @@ private: if (!aim2_available()) { throw std::runtime_error{"aim2 is not available, setup it first"}; } - static auto m2 = db2{aim2_game_dir / "data" / "quest", 1251}.open().to_map(); + static auto m2 = db2{aim2_game_dir / "data" / "quest"}.open().to_map(1251); return m2; } bool aim2_available() const { @@ -503,16 +503,16 @@ private: return backup; } db_wrapper open_db(auto &&name, int db_codepage) { - auto d = db2{get_data_dir() / name, db_codepage}; + auto d = db2{get_data_dir() / name}; auto files = d.open().get_files(); for (auto &&f : files) { backup_or_restore_once(f); files_to_distribute.insert(f); } db_wrapper w; - w.m = d.open().to_map(); + w.m = d.open().to_map(db_codepage); w.fn = d.fn; - w.codepage = d.codepage; + w.codepage = db_codepage; return w; } void backup_or_restore_once(const path &fn) { diff --git a/src/common/common.h b/src/common/common.h index 2b9be6c..e4c5495 100644 --- a/src/common/common.h +++ b/src/common/common.h @@ -18,11 +18,25 @@ #pragma once -#include #include +#include +#include -std::string str2utf8(const std::string &codepage_str, int cp = 0); -std::wstring str2utf16(const std::string &codepage_str, int cp = 0); +// MultiByteToWideChar: https://msdn.microsoft.com/en-us/library/windows/desktop/dd319072(v=vs.85).aspx +// code pages: https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756(v=vs.85).aspx +// https://www.ibm.com/docs/en/rational-soft-arch/9.6.1?topic=overview-locales-code-pages-supported +static const std::map code_pages +{ + { "en", 0 }, + { "cz", 1250 }, + { "ru", 1251 }, + { "ge", 1252 }, + { "fr", 1252 }, + { "et", 1257 }, +}; + +std::string str2utf8(const std::string &codepage_str, int cp); +std::wstring str2utf16(const std::string &codepage_str, int cp); std::string str2str(const std::string &codepage_str, int cp_from, int cp_to); diff --git a/src/common/db.cpp b/src/common/db.cpp index db7ee72..858d41b 100644 --- a/src/common/db.cpp +++ b/src/common/db.cpp @@ -147,11 +147,11 @@ void db::open(const path &p) v.load_fields(t, b); } -polygon4::tools::db::processed_db db::process() const +polygon4::tools::db::processed_db db::process(int cp) const { - auto process_string = [](const std::string &s) + auto process_string = [&](const std::string &s) { - return str2utf8(s.c_str()); + return str2utf8(s.c_str(), cp); }; polygon4::tools::db::processed_db pdb; diff --git a/src/common/db.h b/src/common/db.h index 2dc2f72..3b15c2b 100644 --- a/src/common/db.h +++ b/src/common/db.h @@ -107,5 +107,5 @@ struct db void load(const buffer &b); void open(const path &basename); - polygon4::tools::db::processed_db process() const; + polygon4::tools::db::processed_db process(int cp) const; }; diff --git a/src/common/db2.h b/src/common/db2.h index ed5a6a7..5cfcd10 100644 --- a/src/common/db2.h +++ b/src/common/db2.h @@ -25,13 +25,13 @@ #include -std::string utf8_to_dbstr(const char8_t *s, int codepage = 1251) { +std::string utf8_to_dbstr(const char8_t *s, int codepage) { return str2str((const char *)s, CP_UTF8, codepage); } -std::string utf8_to_dbstr(const char *s, int codepage = 1251) { +std::string utf8_to_dbstr(const char *s, int codepage) { return utf8_to_dbstr((const char8_t *)s, codepage); } -std::string utf8_to_dbstr(const std::string &s, int codepage = 1251) { +std::string utf8_to_dbstr(const std::string &s, int codepage) { return utf8_to_dbstr((const char8_t *)s.c_str(), codepage); } @@ -149,7 +149,6 @@ struct db2 { }; path fn; - int codepage{1251}; template struct file { @@ -223,9 +222,9 @@ struct db2 { } } } - void save(const path &fn, int codepage = 1251) { - auto s_to_char20 = [&](char20 &dst, const std::string &in, int codepage = 1251) { - auto s = utf8_to_dbstr(in); + void save(const path &fn, int codepage) { + auto s_to_char20 = [&](char20 &dst, const std::string &in, int codepage) { + auto s = utf8_to_dbstr(in, codepage); if (s.size() + 1 > sizeof(char20)) { throw std::runtime_error{"too long string"}; } @@ -241,7 +240,7 @@ struct db2 { for (auto &&[tn,td] : m) { tab::table &t = tabv; t.id = table_id; - s_to_char20(t.name, tn, 1251); // always 1251 + s_to_char20(t.name, tn, 1251); // always 1251, because latin only letters for (auto &&[_,fd] : td) { for (auto &&[fn,fv] : fd) { @@ -255,7 +254,7 @@ struct db2 { f.table_id = table_id; f.type = ft; ft = (field_type)total_fields; - s_to_char20(f.name, fn, 1251); // always 1251 if we have any field in Russian + s_to_char20(f.name, fn, 1251); // always 1251, because latin only letters } ++table_id; @@ -282,7 +281,7 @@ struct db2 { auto sz = visit(fv, [&](const int &v) { return datv = v; }, [&](const float &v) { return datv = v; }, - [&](const std::string &v) { return datv = utf8_to_dbstr(v); }); + [&](const std::string &v) { return datv = utf8_to_dbstr(v, codepage); }); auto &v = datv.at(-(sizeof(dat::field_value_base) + sz)); v.field_id = (int)fields[tn].find(fn)->second; v.size = sz; @@ -300,9 +299,9 @@ struct db2 { // converts string to utf8 // filters out values with empty name "" - auto to_map() const { - auto prepare_string = [](auto &&in) { - auto s = str2utf8(in); + auto to_map(int cp) const { + auto prepare_string = [&](auto &&in) { + auto s = str2utf8(in, cp); // we have some erroneous table values (records) with spaces // we can trim only field values, but don't do it at the moment //boost::trim(s); diff --git a/src/db_add_language/db_add_language.cpp b/src/db_add_language/db_add_language.cpp index a649999..97a0154 100644 --- a/src/db_add_language/db_add_language.cpp +++ b/src/db_add_language/db_add_language.cpp @@ -41,15 +41,6 @@ #include -// MultiByteToWideChar: https://msdn.microsoft.com/en-us/library/windows/desktop/dd319072(v=vs.85).aspx -// code pages: https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756(v=vs.85).aspx -const std::map code_pages -{ - { "en", 0 }, - { "ru", 1251 }, - { "et", 1257 }, -}; - static int get_cp(const std::string &cp) { auto i = code_pages.find(cp); diff --git a/src/db_extractor2/db_extractor2.cpp b/src/db_extractor2/db_extractor2.cpp index d5d8386..ca03a54 100644 --- a/src/db_extractor2/db_extractor2.cpp +++ b/src/db_extractor2/db_extractor2.cpp @@ -32,6 +32,7 @@ int main(int argc, char *argv[]) { cl::opt db_fn(cl::Positional, cl::desc(""), cl::Required); + cl::opt codepage(cl::Positional, cl::desc(""), cl::Required); cl::ParseCommandLineOptions(argc, argv); @@ -40,12 +41,12 @@ int main(int argc, char *argv[]) if (fn.extension() != ".json") { db2 db{fn}; auto f = db.open(); - auto m = f.to_map(); + auto m = f.to_map(codepage); write_file(path{fn} += ".json", m.to_json().dump(1)); } else { db2::files::db2_internal db; db.load_from_json(fn); - db.save(fn.parent_path() / fn.stem()); + db.save(fn.parent_path() / fn.stem(), codepage); } return 0;