zhanshi/mekhanoidy-tools
1
0
Fork 0
mirror of https://github.com/aimrebirth/tools.git synced 2026-04-14 17:33:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[db2] Update.

Browse source
This commit is contained in:
lzwdgc 2024-04-02 01:34:43 +03:00
parent 8ac4cdc8a8
commit 1124c73b14
2 changed files with 130 additions and 256 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

306
src/common/db2.h
View file

@ -21,6 +21,8 @@
#include <primitives/templates2/type_name.h> #include <primitives/templates2/type_name.h>
#include <variant>
struct db2 { struct db2 {
using char20 = char[0x20]; using char20 = char[0x20];
@ -56,6 +58,11 @@ struct db2 {
auto base = (field *)(table_base + n_tables); auto base = (field *)(table_base + n_tables);
return std::span{base, base + n_fields}; return std::span{base, base + n_fields};
} }
auto fields(int table_id) {
auto table_base = (table *)(&n_fields + 1);
auto base = (field *)(table_base + n_tables);
return std::span{base, base + n_fields} | std::views::filter([=](auto &v){return v.table_id == table_id;});
}
}; };
// table values (index) // table values (index)
struct ind { struct ind {
@ -72,6 +79,10 @@ struct db2 {
auto base = (value *)(&n_values + 1); auto base = (value *)(&n_values + 1);
return std::span{base, base + n_values}; return std::span{base, base + n_values};
} }
auto values(int table_id) {
auto base = (value *)(&n_values + 1);
return std::span{base, base + n_values} | std::views::filter([=](auto &v){return v.table_id == table_id;});
}
}; };
// field values // field values
struct dat { struct dat {
@ -100,207 +111,128 @@ struct db2 {
}; };
// actual db // actual db
struct files { struct files {
template <typename T, typename V>
struct setter {
T field;
V value;
void operator=(auto &&v) {
value.set_field(field, v);
}
};
struct table {
files &f;
db2::tab::table t;
auto find_field(auto &&name, auto ftype) {
auto fields = f.tab_.data->fields();
auto field_db_cp = f.db.utf8_to_dbstr(name);
auto itf = std::ranges::find_if(fields, [&](auto &v) {
return v.table_id == t.id && field_db_cp == v.name;
});
if (itf == fields.end()) {
if (field_db_cp.size() + 1 > sizeof(tab::field::name)) {
throw std::runtime_error{"too long field name: "s + field_db_cp};
}
auto p = f.tab_.f.alloc_raw(f.tab_.f.sz + sizeof(tab::field));
auto &newfield = *(tab::field *)p;
memset(&newfield, 0, sizeof(newfield));
newfield.table_id = t.id;
newfield.id = ++f.tab_.data->n_fields;
strcpy(newfield.name, field_db_cp.data());
newfield.type = (decltype(newfield.type))ftype;
return newfield;
}
if (itf->type != ftype) {
throw std::runtime_error{"field type mismatch: "s + field_db_cp};
}
return *itf;
}
auto find_value(auto &&name) {
auto values = f.ind_.data->values();
auto value_db_cp = f.db.utf8_to_dbstr(name);
auto itv = std::ranges::find_if(values, [&](auto &v) {
return v.table_id == t.id && value_db_cp == v.name;
});
if (itv == values.end()) {
db2::ind::value i{};
i.table_id = t.id;
if (value_db_cp.size() + 1 > sizeof(i.name)) {
throw std::runtime_error{"too long value name: "s + value_db_cp};
}
memcpy(i.name, value_db_cp.data(), value_db_cp.size());
i.offset = f.dat_.f.sz;
++f.ind_.data->n_values;
auto p = f.ind_.f.alloc_raw(f.ind_.f.sz + sizeof(i));
memcpy(p, &i, sizeof(i));
return value{*this,*(db2::ind::value *)p};
}
return value{*this,*itv};
}
auto operator()(auto &&name) {
return find_value(name);
}
auto operator()(auto &&vname, auto &&fname) {
auto value = find_value(vname);
if constexpr (std::is_convertible_v<decltype(fname), const char *>) {
return setter<const char *, decltype(value)>{fname, value};
} else {
return setter<std::string, decltype(value)>{fname, value};
}
}
};
struct value {
table t;
db2::ind::value &v;
static auto field_type(auto &&v) {
if constexpr (std::same_as<std::decay_t<decltype(v)>, int>) {
return field_type::integer;
} else if constexpr (std::same_as<std::decay_t<decltype(v)>, float>) {
return field_type::float_;
} else {
return field_type::string;
}
}
void set_field(auto &&name, auto &&v) {
using T = std::decay_t<decltype(v)>;
auto f = t.find_field(name, field_type(v));
if (f.type != field_type(v)) {
throw std::runtime_error{"field type mismatch: "s + t.f.db.utf8_to_dbstr(name)};
}
dat::field_value_base newfield{f.id};
if constexpr (std::same_as<std::decay_t<decltype(v)>, int> ||
std::same_as<std::decay_t<decltype(v)>, float>) {
newfield.size = sizeof(T);
} else {
newfield.size = t.f.db.utf8_to_dbstr(v).size() + 1;
}
uint32_t newfieldsize = sizeof(newfield) + newfield.size;
std::vector<uint8_t> data(this->v.size + newfieldsize);
auto dp = data.data();
auto base = t.f.dat_.f.p + this->v.offset;
auto p = base;
while (p < base + this->v.size) {
auto &header = *(dat::field_value_base *)p;
auto len = sizeof(header) + header.size;
if (header.field_id != f.id) {
memcpy(dp, p, len);
dp += len;
} else {
if constexpr (std::same_as<std::decay_t<decltype(v)>, int> ||
std::same_as<std::decay_t<decltype(v)>, float>) {
if (header.size == newfield.size && memcmp(p + sizeof(header), &v, sizeof(v)) == 0) {
return;
}
} else {
if (header.size == newfield.size &&
strcmp((const char *)p + sizeof(header), t.f.db.utf8_to_dbstr(v).data()) == 0) {
return;
}
}
}
p += len;
}
*(dat::field_value_base *)dp = newfield;
dp += sizeof(newfield);
if constexpr (std::same_as<std::decay_t<decltype(v)>, int> ||
std::same_as<std::decay_t<decltype(v)>, float>) {
*(T *)dp = v;
dp += sizeof(v);
} else {
auto s = t.f.db.utf8_to_dbstr(v);
memcpy(dp, s.data(), s.size());
dp += s.size() + 1;
}
auto reallen = dp - data.data();
this->v.size = reallen;
this->v.offset = t.f.dat_.f.sz;
memcpy(t.f.dat_.f.alloc_raw(t.f.dat_.f.sz + reallen), data.data(), reallen);
}
auto operator()(auto &&name) {
if constexpr (std::is_convertible_v<decltype(name), const char *>) {
return setter<const char *, value>{name, *this};
} else {
return setter<std::string, value>{name, *this};
}
}
};
db2 &db; db2 &db;
file<tab> tab_; file<tab> tab_;
file<ind> ind_; file<ind> ind_;
file<dat> dat_; file<dat> dat_;
files(auto &&db, auto &&base) : db{db}, tab_{base}, ind_{base}, dat_{base} {} files(auto &&db, auto &&base) : db{db}, tab_{base}, ind_{base}, dat_{base} {}
void alloc(auto sz) {
tab_.f.alloc_raw(sz);
ind_.f.alloc_raw(sz);
dat_.f.alloc_raw(sz);
}
auto get_files() const { auto get_files() const {
return std::set<path>{tab_.fn,ind_.fn,dat_.fn}; return std::set<path>{tab_.fn,ind_.fn,dat_.fn};
} }
auto find_table(auto &&name) {
auto tbl = tab_.data->tables();
auto table_db_cp = db.utf8_to_dbstr(name);
auto it = std::ranges::find_if(tbl, [&](auto &v) {
return v.name == table_db_cp;
});
if (it == tbl.end()) {
if (table_db_cp.size() + 1 > sizeof(tab::table::name)) {
throw std::runtime_error{"too long table name: "s + table_db_cp};
}
tab_.f.alloc_raw(tab_.f.sz + sizeof(tab::table));
auto base = tab_.f.p + sizeof(tab) + tab_.data->n_tables * sizeof(tab::table);
memmove(base + sizeof(tab::table), base, tab_.f.sz - (base - tab_.f.p + sizeof(tab::table)));
auto &newtab = *(tab::table *)base;
memset(&newtab, 0, sizeof(newtab));
newtab.id = ++tab_.data->n_tables;
strcpy(newtab.name, table_db_cp.data());
return table{*this, newtab};
}
return table{*this,*it};
}
// [] not in msvc yet struct db2_internal {
auto operator()(auto &&tname) { using db2_memory_value = std::variant<std::string, int, float>;
return find_table(tname); using db2_memory = std::map<std::string, std::map<std::string, std::map<std::string, db2_memory_value>>>;
}
auto operator()(auto &&tname, auto &&vname, auto &&fname) { db2_memory m;
auto tbl = find_table(tname);
auto value = tbl.find_value(vname); auto &operator[](this auto &&d, const std::string &s) {
if constexpr (std::is_convertible_v<decltype(fname), const char *>) { return d.m[s];
return setter<const char *, decltype(value)>{fname, value};
} else {
return setter<std::string, decltype(value)>{fname, value};
} }
void save() {
}
};
// converts string to utf8, trims them
auto to_map() const {
auto prepare_string = [](auto &&in) {
auto s = str2utf8(in);
boost::trim(s);
return s;
};
db2_internal m;
auto tbl = tab_.data->tables();
for (auto &&t : tbl) {
auto &jt = m[prepare_string(t.name)];
auto fields = tab_.data->fields(t.id);
for (auto &&v : ind_.data->values(t.id)) {
auto vn = prepare_string(v.name);
if (jt.contains(vn)) {
throw std::logic_error{"duplicate"};
}
auto &jv = jt[vn];
auto p = dat_.f.p + v.offset;
auto max = p + v.size;
while (p < max) {
auto vb = (db2::dat::field_value_base *)p;
p += sizeof(db2::dat::field_value_base);
auto f = std::ranges::find_if(fields, [&](auto &f) { return f.id == vb->field_id; });
if (f == fields.end()) {
throw std::logic_error{"unknown field"};
}
auto fn = prepare_string(f->name);
switch (f->type) {
case db2::field_type::integer:
jv[fn] = *(int *)p;
break;
case db2::field_type::float_:
jv[fn] = *(float *)p;
break;
case db2::field_type::string:
jv[fn] = prepare_string((const char *)p);
break;
default:
throw std::logic_error{"bad type"};
}
p += vb->size;
}
}
}
return m;
} }
/*auto to_json() const {
auto prepare_string = [](auto &&in) {
auto s = str2utf8(in);
boost::trim(s);
return s;
};
auto tbl = tab_.data->tables();
nlohmann::json ja;
for (auto &&t : tbl) {
auto &jt = ja[prepare_string(t.name)];
auto fields = tab_.data->fields(t.id);
for (auto &&v : ind_.data->values(t.id)) {
auto vn = prepare_string(v.name);
if (jt.contains(vn)) {
throw std::logic_error{"duplicate"};
}
auto &jv = jt[vn];
auto p = dat_.f.p + v.offset;
auto max = p + v.size;
while (p < max) {
auto vb = (db2::dat::field_value_base *)p;
p += sizeof(db2::dat::field_value_base);
auto f = std::ranges::find_if(fields, [&](auto &f) {
return f.id == vb->field_id;
});
if (f == fields.end()) {
throw std::logic_error{"unknown field"};
}
auto fn = prepare_string(f->name);
switch (f->type) {
case db2::field_type::integer:
jv[fn] = *(int *)p;
break;
case db2::field_type::float_:
jv[fn] = *(float *)p;
break;
case db2::field_type::string:
jv[fn] = prepare_string((const char *)p);
break;
default:
throw std::logic_error{"bad type"};
}
p += vb->size;
}
}
}
}*/
}; };
auto alloc() {
files{*this,fn}.alloc(0);
}
auto open() { auto open() {
return files{*this, fn}; return files{*this, fn};
} }

80
src/db_extractor2/db_extractor2.cpp
View file

@ -24,6 +24,7 @@
#include <primitives/sw/main.h> #include <primitives/sw/main.h>
#include <primitives/sw/settings.h> #include <primitives/sw/settings.h>
#include <primitives/sw/cl.h> #include <primitives/sw/cl.h>
#include <primitives/templates2/overload.h>
#include <nlohmann/json.hpp> #include <nlohmann/json.hpp>
#include <fstream> #include <fstream>
@ -38,84 +39,25 @@ int main(int argc, char *argv[])
db2 db{db_fn}; db2 db{db_fn};
auto f = db.open(); auto f = db.open();
auto tbl = f.to_map();
/*auto tbl2 = f.find_table(u8"Ãëàéäåðû");
auto valuexxx = tbl2.find_value("GL_S3_PS_FINDER2");
auto valuexxx2 = tbl2.find_value("GL_TEST_XXX");
valuexxx2.set_field("NAME", "X");
valuexxx2.set_field("NAME", "X");
valuexxx2.set_field("NAME", "X");
valuexxx2.set_field("NAME", "X");
valuexxx2.set_field("NAME", "X");
valuexxx2.set_field("NA", "X");
valuexxx2.set_field("VW", 5);
valuexxx2.set_field("VW2", 5.3f);
valuexxx2.set_field("VW2", 6.3f);
valuexxx2.set_field("VW2", 6.3f);
valuexxx2.set_field("VW2", 6.3f);
valuexxx2.set_field("VW2", 6.3f);
f(u8"Ãëàéäåðû", "GL_S3_PS_FINDER2", "NAME") = "xx";
f(u8"Ãëàéäåðû", "GL_S3_PS_FINDER2", "VW2") = 4.3f;*/
auto tbl = f.tab_.data->tables();
auto fields = f.tab_.data->fields();
auto values = f.ind_.data->values();
auto prepare_string = [](auto &&in) {
auto s = str2utf8(in);
boost::trim(s);
return s;
};
nlohmann::json ja; nlohmann::json ja;
for (auto &&t : tbl) { for (auto &&[tn,t] : tbl) {
auto &jt = ja[prepare_string(t.name)]; auto &jt = ja[tn];
for (auto &&v : values | std::views::filter([&](auto &v){return v.table_id == t.id;})) { for (auto &&[vn,v] : t) {
auto vn = prepare_string(v.name);
if (jt.contains(vn)) {
throw std::logic_error{"duplicate"};
}
auto &jv = jt[vn]; auto &jv = jt[vn];
auto max = f.dat_.f.p + v.offset + v.size; for (auto &&[fn, fv] : v) {
auto p = f.dat_.f.p + v.offset; std::visit(overload{
while (p < max) { [&](const int &v){jv[fn] = v;},
auto vb = (db2::dat::field_value_base*)p; [&](const float &v){jv[fn] = v;},
p += sizeof(db2::dat::field_value_base); [&](const std::string &v){jv[fn] = v;},
auto f = std::ranges::find_if(fields, [&](auto &f){return f.table_id == t.id && f.id == vb->field_id;}); }, fv);
if (f == fields.end()) {
continue;
}
switch (f->type) {
case db2::field_type::integer: {
auto fv = (int*)p;
p += vb->size;
jv[prepare_string(f->name)] = *fv;
break;
}
case db2::field_type::float_: {
auto fv = (float*)p;
p += vb->size;
jv[prepare_string(f->name)] = *fv;
break;
}
case db2::field_type::string: {
auto fv = (const char*)p;
p += vb->size;
jv[prepare_string(f->name)] = prepare_string(fv);
break;
}
default: {
break;
}
}
} }
} }
} }
write_file(path{db_fn} += ".json", ja.dump(1)); write_file(path{db_fn} += ".json", ja.dump(1));
db2 x{path{db_fn} += "new"}; db2 x{path{db_fn} += "new"};
x.alloc();
auto newdb = x.open(); auto newdb = x.open();
for (auto &&[t,vals] : ja.items()) { for (auto &&[t,vals] : ja.items()) {
for (auto &&[v,fields] : vals.items()) { for (auto &&[v,fields] : vals.items()) {