From b586253e4c59beac9dc9552550302ebfc9b6c82a Mon Sep 17 00:00:00 2001 From: lzwdgc Date: Fri, 16 Feb 2024 16:55:07 +0300 Subject: [PATCH] Improve db2 api. --- src/common/db2.h | 275 ++++++++++++++++++++++------ src/db_extractor2/db_extractor2.cpp | 31 +++- 2 files changed, 243 insertions(+), 63 deletions(-) diff --git a/src/common/db2.h b/src/common/db2.h index 77bf99b..41b3224 100644 --- a/src/common/db2.h +++ b/src/common/db2.h @@ -19,6 +19,8 @@ #include "common.h" #include "mmap.h" +#include + struct db2 { using char20 = char[0x20]; @@ -82,43 +84,204 @@ struct db2 { }; path fn; - primitives::templates2::mmap_file fdat, find, ftab; - tab *tab_{}; - ind *ind_{}; - dat *dat_{}; int codepage{1251}; - db2() = default; - db2(const path &fn) : fn{fn} { - open(fn, primitives::templates2::mmap_file::ro{}); - } + template + struct file { + path fn; + primitives::templates2::mmap_file f; + T *data; - void open(const path &fn, auto mode) { - close(); - this->fn = fn; - fdat.open(path{fn} += ".dat", mode); - find.open(path{fn} += ".ind", mode); - ftab.open(path{fn} += ".tab", mode); + file(auto &&base) : fn{path{base} += "."s} { + fn += type_name(); + f.open(fn, primitives::templates2::mmap_file::rw{}); + data = (T *)f.p; + } + }; + // actual db + struct files { + struct table { + files &f; + db2::tab::table t; - dat_ = (dat *)find.p; - ind_ = (ind *)find.p; - tab_ = (tab *)ftab.p; - } - void open(auto mode) { - open(fn, mode); - } - void close() { - fdat.close(); - find.close(); - ftab.close(); - } + auto find_field(auto &&name, auto ftype) { + auto fields = f.tab_.data->fields(); + auto field_db_cp = f.db.utf8_to_dbstr(name); + auto itf = std::ranges::find_if(fields, [&](auto &v) { + return v.table_id == t.id && field_db_cp == v.name; + }); + if (itf == fields.end()) { + if (field_db_cp.size() + 1 > sizeof(tab::field::name)) { + throw std::runtime_error{"too long field name: "s + field_db_cp}; + } + auto p = f.tab_.f.alloc_raw(f.tab_.f.sz + sizeof(tab::field)); + auto &newfield = *(tab::field *)p; + memset(&newfield, 0, sizeof(newfield)); + newfield.table_id = t.id; + newfield.id = ++f.tab_.data->n_fields; + strcpy(newfield.name, field_db_cp.data()); + newfield.type = (decltype(newfield.type))ftype; + return newfield; + } + if (itf->type != ftype) { + throw std::runtime_error{"field type mismatch: "s + field_db_cp}; + } + return *itf; + } + auto find_value(auto &&name) { + auto values = f.ind_.data->values(); + auto value_db_cp = f.db.utf8_to_dbstr(name); + auto itv = std::ranges::find_if(values, [&](auto &v) { + return v.table_id == t.id && value_db_cp == v.name; + }); + if (itv == values.end()) { + db2::ind::value i{}; + i.table_id = t.id; + if (value_db_cp.size() + 1 > sizeof(i.name)) { + throw std::runtime_error{"too long value name: "s + value_db_cp}; + } + memcpy(i.name, value_db_cp.data(), value_db_cp.size()); + i.offset = f.dat_.f.sz; + ++f.ind_.data->n_values; + auto p = f.ind_.f.alloc_raw(f.ind_.f.sz + sizeof(i)); + memcpy(p, &i, sizeof(i)); + return value{*this,*(db2::ind::value *)p}; + } + return value{*this,*itv}; + } + }; + struct value { + table t; + db2::ind::value &v; - void add_value(std::string_view table, std::string_view value, auto && ... fields1) { - auto tbl = tab_->tables(); - auto fields = tab_->fields(); - auto values = ind_->values(); + static auto field_type(auto &&v) { + if constexpr (std::same_as, int>) { + return field_type::integer; + } else if constexpr (std::same_as, float>) { + return field_type::float_; + } else { + return field_type::string; + } + } + void set_field(auto &&name, auto &&v) { + using T = std::decay_t; + auto f = t.find_field(name, field_type(v)); + dat::field_value_base newfield{f.id}; + if constexpr (std::same_as, int> || + std::same_as, float>) { + newfield.size = sizeof(T); + } else { + newfield.size = t.f.db.utf8_to_dbstr(v).size() + 1; + } + uint32_t newfieldsize = sizeof(newfield) + newfield.size; + std::vector data(this->v.size + newfieldsize); + auto dp = data.data(); + auto base = t.f.dat_.f.p + this->v.offset; + auto p = base; + while (p < base + this->v.size) { + auto &header = *(dat::field_value_base *)p; + auto len = sizeof(header) + header.size; + if (header.field_id != f.id) { + memcpy(dp, p, len); + dp += len; + } else { + if constexpr (std::same_as, int> || + std::same_as, float>) { + if (header.size == newfield.size && memcmp(p + sizeof(header), &v, sizeof(v)) == 0) { + return; + } + } else { + if (header.size == newfield.size && + strcmp((const char *)p + sizeof(header), t.f.db.utf8_to_dbstr(v).data()) == 0) { + return; + } + } + } + p += len; + } + *(dat::field_value_base *)dp = newfield; + dp += sizeof(newfield); + if constexpr (std::same_as, int> || + std::same_as, float>) { + *(T *)dp = v; + dp += sizeof(v); + } else { + auto s = t.f.db.utf8_to_dbstr(v); + memcpy(dp, s.data(), s.size()); + dp += s.size() + 1; + } + auto reallen = dp - data.data(); + this->v.size = reallen; + this->v.offset = t.f.dat_.f.sz; + memcpy(t.f.dat_.f.alloc_raw(t.f.dat_.f.sz + reallen), data.data(), reallen); + } + }; + template + struct setter { + T field; + value value; + void operator=(auto &&v) { + value.set_field(field, v); + } + }; - auto calc_fields_size = [&](this auto &&f, std::string_view field_name, auto &&n, auto &&v, auto &&...fields) { + db2 &db; + file tab_; + file ind_; + file dat_; + + files(auto &&db, auto &&base) : db{db}, tab_{base}, ind_{base}, dat_{base} {} + + auto get_files() const { + return std::set{tab_.fn,ind_.fn,dat_.fn}; + } + auto find_table(auto &&name) { + auto tbl = tab_.data->tables(); + auto table_db_cp = db.utf8_to_dbstr(name); + auto it = std::ranges::find_if(tbl, [&](auto &v) { + return v.name == table_db_cp; + }); + if (it == tbl.end()) { + if (table_db_cp.size() + 1 > sizeof(tab::table::name)) { + throw std::runtime_error{"too long table name: "s + table_db_cp}; + } + tab_.f.alloc_raw(tab_.f.sz + sizeof(tab::table)); + auto base = tab_.f.p + sizeof(tab) + tab_.data->n_tables * sizeof(tab::table); + memmove(base + sizeof(tab::table), base, tab_.f.sz - (base - tab_.f.p + sizeof(tab::table))); + auto &newtab = *(tab::table *)base; + memset(&newtab, 0, sizeof(newtab)); + newtab.id = ++tab_.data->n_tables; + strcpy(newtab.name, table_db_cp.data()); + return table{*this, newtab}; + } + return table{*this,*it}; + } + + // [] not in msvc yet + auto operator()(auto &&tname, auto &&vname, auto &&fname) { + auto tbl = find_table(tname); + auto value = tbl.find_value(vname); + if constexpr (std::is_convertible_v) { + return setter{fname, value}; + } else { + return setter{fname, value}; + } + } + }; + + auto open() { + return files{*this,fn}; + } + void add_value(auto &&table, auto &&value, auto && ... fields1) { + auto f = open(); + auto tbl = f.tab_.data->tables(); + auto fields = f.tab_.data->fields(); + auto values = f.ind_.data->values(); + + auto table_db_cp = utf8_to_dbstr(table); + auto value_db_cp = utf8_to_dbstr(value); + + auto calc_fields_size = [&](this auto &&f, auto &&field_name, auto &&n, auto &&v, auto &&...fields) { if (field_name == n) { if constexpr (std::same_as, int>) { return sizeof(db2::dat::field_value_base) + sizeof(int); @@ -140,7 +303,7 @@ struct db2 { return sizeof(db2::dat::field_value_base) + 1; } }; - auto write_fields = [&](this auto &&f, auto &&p, auto &&field, std::string_view field_name, auto &&n, auto &&v, auto &&...fields) { + auto write_fields = [&](this auto &&f, auto &&p, auto &&field, auto &&field_name, auto &&n, auto &&v, auto &&...fields) { if (field_name == n) { if constexpr (std::same_as, int>) { if (field.type != db2::field_type::integer) { @@ -199,17 +362,17 @@ struct db2 { } }; - auto it = std::ranges::find_if(tbl, [&](auto &v){return v.name == table;}); + auto it = std::ranges::find_if(tbl, [&](auto &v){return v.name == table_db_cp;}); if (it == tbl.end()) { - throw std::runtime_error{"no such table"}; + throw std::runtime_error{"no such table: "s + table_db_cp}; } auto &t = *it; - auto itv = std::ranges::find_if(values, [&](auto &v){return v.table_id == t.id && value == v.name;}); + auto itv = std::ranges::find_if(values, [&](auto &v){return v.table_id == t.id && value_db_cp == v.name;}); if (itv == values.end()) { db2::ind::value i{}; i.table_id = t.id; - memcpy(i.name, value.data(), value.size()); - i.offset = fdat.sz; + memcpy(i.name, value_db_cp.data(), value_db_cp.size()); + i.offset = f.dat_.f.sz; for (auto &&f : fields) { if (f.table_id != t.id) { continue; @@ -218,11 +381,11 @@ struct db2 { i.size += calc_fields_size(fn, fields1...); } - ++ind_->n_values; - auto p = find.alloc_raw(find.sz + sizeof(i)); + ++f.ind_.data->n_values; + auto p = f.ind_.f.alloc_raw(f.ind_.f.sz + sizeof(i)); memcpy(p, &i, sizeof(i)); - p = fdat.alloc_raw(fdat.sz + i.size); + p = f.dat_.f.alloc_raw(f.dat_.f.sz + i.size); for (auto &&f : fields) { if (f.table_id != t.id) { continue; @@ -233,22 +396,15 @@ struct db2 { } } template - void edit_value(std::string_view table, std::string_view value, std::string_view field, const T &field_value) { - auto tbl = tab_->tables(); - auto fields = tab_->fields(); - auto values = ind_->values(); + void edit_value(auto &&table, auto &&value, auto &&field, const T &field_value) { + auto f = open(); + auto fields = f.tab_.data->fields(); + auto values = f.ind_.data->values(); - auto table_db_cp = utf8_to_dbstr(table.data()); - auto value_db_cp = utf8_to_dbstr(value.data()); - auto field_db_cp = utf8_to_dbstr(field.data()); + auto value_db_cp = utf8_to_dbstr(value); + auto field_db_cp = utf8_to_dbstr(field); - auto it = std::ranges::find_if(tbl, [&](auto &v) { - return table_db_cp == v.name; - }); - if (it == tbl.end()) { - throw std::runtime_error{"no such table: "s + table_db_cp}; - } - auto &t = *it; + /*auto t = f.find_table(table); auto itv = std::ranges::find_if(values, [&](auto &v) { return v.table_id == t.id && value_db_cp == v.name; }); @@ -262,8 +418,8 @@ struct db2 { throw std::runtime_error{"no such field: "s + field_db_cp}; } - auto p = fdat.p + itv->offset; - while (p < fdat.p + itv->offset + itv->size) { + auto p = f.dat_.f.p + itv->offset; + while (p < f.dat_.f.p + itv->offset + itv->size) { auto &header = *(dat::field_value_base*)p; p += sizeof(header); if (header.field_id == itf->id) { @@ -281,12 +437,15 @@ struct db2 { return; } p += header.size; - } + }*/ throw std::runtime_error{"no such field"}; } private: std::string utf8_to_dbstr(const char *s) const { - return str2str(s, CP_UTF8, codepage); + return utf8_to_dbstr((const char8_t *)s); + } + std::string utf8_to_dbstr(const char8_t *s) const { + return str2str((const char *)s, CP_UTF8, codepage); } }; diff --git a/src/db_extractor2/db_extractor2.cpp b/src/db_extractor2/db_extractor2.cpp index d9bc315..aec1c03 100644 --- a/src/db_extractor2/db_extractor2.cpp +++ b/src/db_extractor2/db_extractor2.cpp @@ -36,9 +36,30 @@ int main(int argc, char *argv[]) cl::ParseCommandLineOptions(argc, argv); db2 db{db_fn}; - auto tbl = db.tab_->tables(); - auto fields = db.tab_->fields(); - auto values = db.ind_->values(); + auto f = db.open(); + + auto tbl2 = f.find_table(u8"Глайдеры"); + auto valuexxx = tbl2.find_value("GL_S3_PS_FINDER2"); + auto valuexxx2 = tbl2.find_value("GL_TEST_XXX"); + valuexxx2.set_field("NAME", "X"); + valuexxx2.set_field("NAME", "X"); + valuexxx2.set_field("NAME", "X"); + valuexxx2.set_field("NAME", "X"); + valuexxx2.set_field("NAME", "X"); + valuexxx2.set_field("NA", "X"); + valuexxx2.set_field("VW", 5); + valuexxx2.set_field("VW2", 5.3f); + valuexxx2.set_field("VW2", 6.3f); + valuexxx2.set_field("VW2", 6.3f); + valuexxx2.set_field("VW2", 6.3f); + valuexxx2.set_field("VW2", 6.3f); + + f(u8"Глайдеры", "GL_S3_PS_FINDER2", "NAME") = "xx"; + f(u8"Глайдеры", "GL_S3_PS_FINDER2", "VW2") = 4.3f; + + auto tbl = f.tab_.data->tables(); + auto fields = f.tab_.data->fields(); + auto values = f.ind_.data->values(); std::string spaceval(4, ' '); std::string spacefield(8, ' '); @@ -46,8 +67,8 @@ int main(int argc, char *argv[]) std::println("{}:", t.name); for (auto &&v : values | std::views::filter([&](auto &v){return v.table_id == t.id;})) { std::println("{}{}:", spaceval, v.name); - auto max = db.fdat.p + v.offset + v.size; - auto p = db.fdat.p + v.offset; + auto max = f.dat_.f.p + v.offset + v.size; + auto p = f.dat_.f.p + v.offset; while (p < max) { auto vb = (db2::dat::field_value_base*)p; p += sizeof(db2::dat::field_value_base);