From 763f4aeb95011d6fdb470bfa9c2e153633800191 Mon Sep 17 00:00:00 2001 From: lzwdgc Date: Fri, 2 Feb 2024 23:21:09 +0300 Subject: [PATCH] [unpaker] Unify m1 and m2 unpakers. --- src/common/common.h | 18 ++- src/common/types.h | 27 +++++ src/unpaker/decode.h | 13 +-- src/unpaker/pak.cpp | 169 --------------------------- src/unpaker/pak.h | 95 --------------- src/unpaker/unpaker.cpp | 206 ++++++++++++++++++++++++++------- src/unpaker2/unpaker2.cpp | 236 -------------------------------------- sw.cpp | 3 +- 8 files changed, 215 insertions(+), 552 deletions(-) delete mode 100644 src/unpaker/pak.cpp delete mode 100644 src/unpaker/pak.h delete mode 100644 src/unpaker2/unpaker2.cpp diff --git a/src/common/common.h b/src/common/common.h index fed103c..bb1e9c3 100644 --- a/src/common/common.h +++ b/src/common/common.h @@ -18,7 +18,23 @@ #pragma once -#include +#include +#include std::string str2utf8(const std::string &codepage_str, int cp = 0); std::wstring str2utf16(const std::string &codepage_str, int cp = 0); + +struct progress_bar { + const size_t max_elements; + const int displaylen; + int displaycur{}; + int i{}; + + void step() { + auto progress_bar_pos = std::round((double)++i / max_elements * displaylen); + for (int i = displaycur; i < progress_bar_pos; ++i) { + std::cout << "#"; + } + displaycur = progress_bar_pos; + } +}; diff --git a/src/common/types.h b/src/common/types.h index cea88c4..ec72cc7 100644 --- a/src/common/types.h +++ b/src/common/types.h @@ -324,3 +324,30 @@ struct ModificatorMask uint16_t : 16; }; + +#pragma pack(push, 1) +struct pak { + uint32_t magic; + uint16_t unk0; + uint32_t n_files; + uint32_t n_blocks; + uint32_t block_size; + uint32_t unk1; +}; +struct file_description { + const char name[0x50]; + uint32_t offset; + uint32_t size; +}; +struct segment { + enum decode_algorithm : uint32_t { + none = 0x0, + // none = 0x1, // ? + }; + + // some file offset? trash? crc? m1 has zlib crc table (png)? + uint32_t unk1; + decode_algorithm algorithm; + uint32_t offset; +}; +#pragma pack(pop) diff --git a/src/unpaker/decode.h b/src/unpaker/decode.h index a95f8c6..a66eeb8 100644 --- a/src/unpaker/decode.h +++ b/src/unpaker/decode.h @@ -57,7 +57,7 @@ char decode_f2(char *input, int size, char *output) } template -auto decode_rle(const T *input, int size, T *output, auto f_cmp_indicator) { +auto decode_rle(const T *input, int size, T *output) { if (size < 2) { return (uint8_t *)output; } @@ -65,7 +65,8 @@ auto decode_rle(const T *input, int size, T *output, auto f_cmp_indicator) { const auto rle_indicator = (uint8_t)*input++; while (1) { auto c = *input++; - if (f_cmp_indicator(c) != rle_indicator) { + //msvc bad warn. workaround vvvvvvvv + if ((sizeof(T) == 1 ? c : ((uint16_t)c >> 8)) != rle_indicator) { *output++ = c; } else { uint32_t count = sizeof(T) == 1 ? *input++ : (c & 0xFF); @@ -83,11 +84,3 @@ auto decode_rle(const T *input, int size, T *output, auto f_cmp_indicator) { } } } - -auto decode_rle(const uint16_t *input, int size, uint16_t *output) { - return decode_rle(input, size, output, [](auto c){return c >> 8;}); -} - -auto decode_rle(const uint8_t *input, const int size, uint8_t *output) { - return decode_rle(input, size, output, [](auto c){return c;}); -} diff --git a/src/unpaker/pak.cpp b/src/unpaker/pak.cpp deleted file mode 100644 index 947efb4..0000000 --- a/src/unpaker/pak.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * AIM 1 unpaker - * Copyright (C) 2015 lzwdgc - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include "pak.h" - -#include -#include -#include -#include - -#include "decode.h" - -#define FREAD(var) fread(&var, 1, sizeof(var), f) - -void header::load(FILE *f) -{ - FREAD(unk1); - FREAD(unk2); - FREAD(number_of_files); - FREAD(unk3); - FREAD(number_of_chunks); - FREAD(chunk_size); - FREAD(unk5); -} - -void record::load(FILE *f) -{ - char n[0x50]; - FREAD(n); - name = n; - FREAD(pos); - FREAD(len); -} - -void record::write(string name, const vector &data) const -{ - name += "\\" + string(this->name); - string dir = name.substr(0, name.rfind('\\')); - system(string("mkdir " + dir + " 2> nul").c_str()); - FILE *f = fopen(name.c_str(), "wb"); - if (!f) - return; - fwrite(data.data(), 1, data.size(), f); - fclose(f); -} - -int record::read(pak *pak, void *output, int size) -{ - int file_start_pos = offset + pos; - int segment = file_start_pos / pak->h.chunk_size; - offset += size; - - pak->segments[segment].decompress(segment); - - auto file_start_pos2 = file_start_pos - segment * pak->h.chunk_size; - auto size3 = size; - if (pak->h.chunk_size - file_start_pos2 < size) - size3 = pak->h.chunk_size - file_start_pos2; - memcpy(output, pak->segments[segment].decoded + file_start_pos2, size3); - - auto size_diff = size - size3; - uint32_t diff = 0; - for (char *out = (char *)output + size3; size_diff > 0; out += diff) - { - segment++; - pak->segments[segment].decompress(segment); - - diff = pak->h.chunk_size; - if (diff >= size_diff) - diff = size_diff; - memcpy(out, pak->segments[segment].decoded, 4 * (diff >> 2)); - size_diff -= diff; - memcpy( - out + 4 * (diff >> 2), - pak->segments[segment].decoded + 4 * (diff >> 2), - diff & 3); - } - return size; -} - -void segment::load_header(FILE *f) -{ - FREAD(unk1); - FREAD(algorithms); - FREAD(offset); -} - -void segment::load_segment() -{ - auto f = file; - - fseek(f, offset, SEEK_SET); - /*if (algorithms == 0) - { - std::cerr << "Something is wrong. Maybe you trying to open aim2 files?\n"; - std::cerr << "They can be opened with SDK extractor.\n"; - throw std::runtime_error("error"); - }*/ - - FREAD(size1); - size2 = size1; - if (algorithms & DA_2) - { - FREAD(size2); - fread(&decoded[0], 1, size2, f); - } - else - { - fread(&encoded[0], 1, size1, f); - } -} - -void segment::decompress(int segment_id) -{ - load_segment(); - - if (algorithms & DA_2) { - decode_f2((char*)decoded, size2, (char*)encoded); - } - if (algorithms & RLE_2_bytes) { - decode_rle((uint16_t *)encoded, size1, (uint16_t *)decoded); - } else if (algorithms & RLE_1_byte) { - decode_rle((uint8_t *)encoded, size1, (uint8_t *)decoded); - } - if (algorithms == None) { - decoded = encoded; - } -} - -void pak::load(FILE *f) -{ - h.load(f); - encoded.resize(h.chunk_size * 4); - decoded.resize(h.chunk_size * 4); - - int n = h.number_of_files; - while (n--) - { - record rec; - rec.load(f); - files[rec.name] = rec; - } - - n = h.number_of_chunks; - while (n--) - { - segment t; - t.load_header(f); - t.file = f; - t.encoded = encoded.data(); - t.decoded = decoded.data(); - segments.push_back(t); - } -} diff --git a/src/unpaker/pak.h b/src/unpaker/pak.h deleted file mode 100644 index 27970d8..0000000 --- a/src/unpaker/pak.h +++ /dev/null @@ -1,95 +0,0 @@ -/* - * AIM 1 unpaker - * Copyright (C) 2015 lzwdgc - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#pragma once - -#include -#include -#include -#include - -using namespace std; - -struct header -{ - uint32_t unk1; - uint16_t unk2; - uint16_t number_of_files; - uint16_t unk3; - uint32_t number_of_chunks; - int32_t chunk_size; - uint32_t unk5; - - void load(FILE *f); -}; - -struct record -{ - std::string name; - uint32_t pos; - uint32_t len; - - // - vector data; - int offset = 0; - - void load(FILE *f); - void write(string name, const vector &data) const; - int read(struct pak *pak, void *output, int size); -}; - -struct segment -{ - enum decode_algorithm : uint32_t - { - None = 0x0, - RLE_2_bytes = 0x1, - RLE_1_byte = 0x2, - DA_1 = 0x4, // not used - DA_2 = 0x8, - }; - - uint32_t unk1; // some file offset? trash? - decode_algorithm algorithms; - uint32_t offset; - - uint32_t size1; - uint32_t size2; - - // - FILE *file = 0; - uint8_t* encoded; - uint8_t* decoded; - - void load_header(FILE *f); - void load_segment(); - void decompress(int segment_id); -}; - -struct pak -{ - header h; - vector segments; - map files; - - // - vector encoded; - vector decoded; - - void load(FILE *f); -}; diff --git a/src/unpaker/unpaker.cpp b/src/unpaker/unpaker.cpp index 63ecfd0..be90d4c 100644 --- a/src/unpaker/unpaker.cpp +++ b/src/unpaker/unpaker.cpp @@ -1,6 +1,6 @@ /* - * AIM 1 unpaker - * Copyright (C) 2015 lzwdgc + * AIM unpaker (for AIM and AIM2 games, AIM:R) + * Copyright (C) 2023 lzwdgc * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,53 +16,181 @@ * along with this program. If not, see . */ -#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include #include +#include +#include +#include +#include -#include "pak.h" +#include +#include -namespace fs = std::filesystem; +#include "decode.h" -void unpak(string fn) -{ - FILE *f = fopen(fn.c_str(), "rb"); - if (!f) - return; - pak p; - p.load(f); +using namespace std; - auto unpack = [&](auto &file) - { - cout << "Unpacking " << file.name << "\n"; - vector buf(file.len); - file.read(&p, &buf[0], file.len); - file.write(fn + ".dir", buf); - }; +void unpack_file(path fn) { + primitives::templates2::mmap_file f{fn}; + stream s{f}; + pak p = s; + auto descs = s.span(p.n_files); + auto segments = s.span(p.n_blocks); + std::vector decoded; + decoded.resize((segments.size() + 1) * p.block_size * 4); + auto pp = decoded.data(); + progress_bar pb{segments.size(), 50}; + for (auto &&seg : segments) { + s.p = f.p + seg.offset; + uint32_t len = s; + auto m2 = [&]() { + enum decode_algorithm : uint32_t { + none = 0x0, + lzo = 0x1, + lzma = 0x2, + rlew = 0x4, // https://moddingwiki.shikadi.net/wiki/Id_Software_RLEW_compression + }; + switch (seg.algorithm) { + case decode_algorithm::none: { + memcpy(pp, s.p, len); + pp += len; + break; + } + case decode_algorithm::lzo: { + size_t outsz; + // use lzo1x_decompress_safe? + auto r2 = lzo1x_decompress(s.p, len, pp, &outsz, 0); + if (r2 != LZO_E_OK) { + throw std::runtime_error{"lzo error"}; + } + pp += outsz; + break; + } + case decode_algorithm::rlew: { + auto base = s.p; + uint16_t flag = s; + while (s.p < base + len) { + uint16_t w = s; + if ((w & 0xFF00) == (flag << 8)) { + uint16_t count = (uint8_t)w; + if (count == 0xFF) { + uint16_t w2 = s; + *(decltype(w2) *)pp = w2; + pp += sizeof(w2); + continue; + } + uint16_t w2 = s; + count += 3; + while (count--) { + *(decltype(w2)*)pp = w2; + pp += sizeof(w2); + } + } else { + *(decltype(w)*)pp = w; + pp += sizeof(w); + } + } + break; + } + case decode_algorithm::lzma: { + uint8_t flags = s; - for (auto &[n,f] : p.files) - unpack(f); - fclose(f); + lzma_stream strm{}; + strm.next_in = s.p; + strm.avail_in = len; + strm.next_out = pp; + strm.avail_out = p.block_size; + + auto r = lzma_lzip_decoder(&strm, 10'000'000, flags); + if (r != LZMA_OK) { + throw std::runtime_error{"lzma error"}; + } + r = lzma_code(&strm, LZMA_RUN); + if (r != LZMA_STREAM_END) { + throw std::runtime_error{"lzma error"}; + } + pp += strm.total_out; + break; + } + default: + throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)}; + } + }; + auto m1 = [&]() { + enum decode_algorithm : uint32_t { + None = 0x0, + RLE_2_bytes = 0x1, + RLE_1_byte = 0x2, + decode_algorithm_1 = 0x4, // not used + decode_algorithm_2 = 0x8, + }; + auto in = s.p; + auto size1 = len; + std::vector vec; + if (seg.algorithm & decode_algorithm_1) { + // if you see this, check in git history decode_f1() + throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)}; + } + if (seg.algorithm & decode_algorithm_2) { + uint32_t size2 = s; + vec.resize(std::max(size2 * 4, p.block_size)); + decode_f2((char *)s.p, size2, (char *)vec.data()); + in = vec.data(); + } + if (seg.algorithm & RLE_2_bytes) { + pp = decode_rle((uint16_t *)in, size1, (uint16_t *)pp); + } else if (seg.algorithm & RLE_1_byte) { + pp = decode_rle((uint8_t *)in, size1, (uint8_t *)pp); + } + if (seg.algorithm == None) { + memcpy(pp, s.p, size1); + } + }; + if (p.magic == 0) { + m1(); + } else { + m2(); + } + pb.step(); + } + std::cout << "\n"; + auto dir = fn += ".dir"; + fs::create_directories(dir); + for (auto &&d : descs) { + auto fn = dir / d.name; + fs::create_directories(fn.parent_path()); + std::cout << "unpacking " << fn << "\n"; + primitives::templates2::mmap_file f{fn, primitives::templates2::mmap_file::rw{}}; + f.alloc_raw(d.size); + memcpy(f.p, decoded.data() + d.offset, d.size); + } } -int main(int argc, char *argv[]) -{ - if (argc != 2) - { - cerr << "Usage: " << argv[0] << " " << "\n"; - return 1; - } - fs::path p = argv[1]; +int main(int argc, char *argv[]) { + cl::opt p(cl::Positional, cl::desc(""), cl::Required); + + cl::ParseCommandLineOptions(argc, argv); + if (fs::is_regular_file(p)) { - unpak(p.string()); + unpack_file(p); } else if (fs::is_directory(p)) { - for (auto &&d : fs::directory_iterator{p}) { - if (d.path().extension() == ".pak") { - std::cout << "processing: " << d.path() << "\n"; - try { - unpak(d.path().string()); - } catch (std::exception &e) { - std::cerr << e.what() << "\n"; - } + auto files = enumerate_files_like(p, ".*\\.pak", false); + for (auto &f : files) { + std::cout << "processing: " << f << "\n"; + try { + unpack_file(f); + } catch (std::exception &e) { + std::cerr << e.what() << "\n"; } } } else { diff --git a/src/unpaker2/unpaker2.cpp b/src/unpaker2/unpaker2.cpp deleted file mode 100644 index 1c6e455..0000000 --- a/src/unpaker2/unpaker2.cpp +++ /dev/null @@ -1,236 +0,0 @@ -/* - * AIM unpaker2 (for AIM2 only at the moment) - * Copyright (C) 2023 lzwdgc - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "../unpaker/decode.h" - -using namespace std; - -#pragma pack(push, 1) -struct pak { - uint32_t magic; - uint16_t unk0; - uint32_t n_files; - uint32_t n_blocks; - uint32_t block_size; - uint32_t unk1; -}; -struct file_description { - const char name[0x50]; - uint32_t offset; - uint32_t size; -}; -struct segment { - // some file offset? trash? crc? m1 has zlib crc table (png)? - uint32_t unk1; - uint32_t algorithm; - uint32_t offset; -}; -#pragma pack(pop) - -struct progress_bar { - const size_t max_elements; - const int displaylen; - int displaycur{}; - int i{}; - - void step() { - auto progress_bar_pos = std::round((double)++i / max_elements * displaylen); - for (int i = displaycur; i < progress_bar_pos; ++i) { - std::cout << "#"; - } - displaycur = progress_bar_pos; - } -}; - -void unpack_file(path fn) { - primitives::templates2::mmap_file f{fn}; - stream s{f}; - pak p = s; - auto descs = s.span(p.n_files); - auto segments = s.span(p.n_blocks); - std::vector decoded; - decoded.resize((segments.size() + 1) * p.block_size * 4); - auto pp = decoded.data(); - progress_bar pb{segments.size(), 50}; - for (auto &&seg : segments) { - s.p = f.p + seg.offset; - uint32_t len = s; - auto m2 = [&]() { - enum decode_algorithm : uint32_t { - none = 0x0, - lzo = 0x1, - lzma = 0x2, - rlew = 0x4, // https://moddingwiki.shikadi.net/wiki/Id_Software_RLEW_compression - }; - switch (seg.algorithm) { - case decode_algorithm::none: { - memcpy(pp, s.p, len); - pp += len; - break; - } - case decode_algorithm::lzo: { - size_t outsz; - // use lzo1x_decompress_safe? - auto r2 = lzo1x_decompress(s.p, len, pp, &outsz, 0); - if (r2 != LZO_E_OK) { - throw std::runtime_error{"lzo error"}; - } - pp += outsz; - break; - } - case decode_algorithm::rlew: { - auto base = s.p; - uint16_t flag = s; - while (s.p < base + len) { - uint16_t w = s; - if ((w & 0xFF00) == (flag << 8)) { - uint16_t count = (uint8_t)w; - if (count == 0xFF) { - uint16_t w2 = s; - *(decltype(w2) *)pp = w2; - pp += sizeof(w2); - continue; - } - uint16_t w2 = s; - count += 3; - while (count--) { - *(decltype(w2)*)pp = w2; - pp += sizeof(w2); - } - } else { - *(decltype(w)*)pp = w; - pp += sizeof(w); - } - } - break; - } - case decode_algorithm::lzma: { - uint8_t flags = s; - - lzma_stream strm{}; - strm.next_in = s.p; - strm.avail_in = len; - strm.next_out = pp; - strm.avail_out = p.block_size; - - auto r = lzma_lzip_decoder(&strm, 10'000'000, flags); - if (r != LZMA_OK) { - throw std::runtime_error{"lzma error"}; - } - r = lzma_code(&strm, LZMA_RUN); - if (r != LZMA_STREAM_END) { - throw std::runtime_error{"lzma error"}; - } - pp += strm.total_out; - break; - } - default: - throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)}; - } - }; - auto m1 = [&]() { - enum decode_algorithm : uint32_t { - None = 0x0, - RLE_2_bytes = 0x1, - RLE_1_byte = 0x2, - decode_algorithm_1 = 0x4, // not used - decode_algorithm_2 = 0x8, - }; - auto in = s.p; - auto size1 = len; - std::vector vec; - if (seg.algorithm & decode_algorithm_1) { - // if you see this, check in git history decode_f1() - throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)}; - } - if (seg.algorithm & decode_algorithm_2) { - uint32_t size2 = s; - vec.resize(std::max(size2 * 4, p.block_size)); - decode_f2((char *)s.p, size2, (char *)vec.data()); - in = vec.data(); - } - if (seg.algorithm & RLE_2_bytes) { - pp = decode_rle((uint16_t *)in, size1, (uint16_t *)pp); - } else if (seg.algorithm & RLE_1_byte) { - pp = decode_rle((uint8_t *)in, size1, (uint8_t *)pp); - } - if (seg.algorithm == None) { - //decoded = encoded; - } - }; - if (p.magic == 0) { - m1(); - } else { - m2(); - } - pb.step(); - } - std::cout << "\n"; - auto dir = fn += ".dir"; - fs::create_directories(dir); - for (auto &&d : descs) { - auto fn = dir / d.name; - fs::create_directories(fn.parent_path()); - std::cout << "unpacking " << fn << "\n"; - primitives::templates2::mmap_file f{fn, primitives::templates2::mmap_file::rw{}}; - f.alloc_raw(d.size); - memcpy(f.p, decoded.data() + d.offset, d.size); - } -} - -int main(int argc, char *argv[]) { - cl::opt p(cl::Positional, cl::desc(""), cl::Required); - - cl::ParseCommandLineOptions(argc, argv); - - if (fs::is_regular_file(p)) { - unpack_file(p); - } else if (fs::is_directory(p)) { - auto files = enumerate_files_like(p, ".*\\.pak", false); - for (auto &f : files) { - std::cout << "processing: " << f << "\n"; - try { - unpack_file(f); - } catch (std::exception &e) { - std::cerr << e.what() << "\n"; - } - } - } else { - throw std::runtime_error("Bad fs object"); - } - return 0; -} diff --git a/sw.cpp b/sw.cpp index c88c420..8028ae8 100644 --- a/sw.cpp +++ b/sw.cpp @@ -53,8 +53,7 @@ void build(Solution &s) add_exe_with_common("tm_converter"); add_exe("name_generator"); add_exe_with_common("save_loader"); - add_exe("unpaker"); - add_exe_with_common("unpaker2") += + add_exe_with_common("unpaker") += "org.sw.demo.oberhumer.lzo.lzo"_dep, "org.sw.demo.xz_utils.lzma"_dep ;