zhanshi/mekhanoidy-tools
1
0
Fork 0
mirror of https://github.com/aimrebirth/tools.git synced 2026-04-15 01:43:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[unpaker] Unify m1 and m2 unpakers.

Browse source
This commit is contained in:
lzwdgc 2024-02-02 23:21:09 +03:00
parent 9146cf6900
commit 763f4aeb95
8 changed files with 215 additions and 552 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/common/common.h
View file

@ -18,7 +18,23 @@
#pragma once
#include <primitives/filesystem.h>
#include <string>
#include <iostream>
std::string str2utf8(const std::string &codepage_str, int cp = 0);
std::wstring str2utf16(const std::string &codepage_str, int cp = 0);
struct progress_bar {
const size_t max_elements;
const int displaylen;
int displaycur{};
int i{};
void step() {
auto progress_bar_pos = std::round((double)++i / max_elements * displaylen);
for (int i = displaycur; i < progress_bar_pos; ++i) {
std::cout << "#";
}
displaycur = progress_bar_pos;
}
};

27
src/common/types.h
View file

@ -324,3 +324,30 @@ struct ModificatorMask
uint16_t : 16;
};
#pragma pack(push, 1)
struct pak {
uint32_t magic;
uint16_t unk0;
uint32_t n_files;
uint32_t n_blocks;
uint32_t block_size;
uint32_t unk1;
};
struct file_description {
const char name[0x50];
uint32_t offset;
uint32_t size;
};
struct segment {
enum decode_algorithm : uint32_t {
none = 0x0,
// none = 0x1, // ?
};
// some file offset? trash? crc? m1 has zlib crc table (png)?
uint32_t unk1;
decode_algorithm algorithm;
uint32_t offset;
};
#pragma pack(pop)

13
src/unpaker/decode.h
View file

@ -57,7 +57,7 @@ char decode_f2(char *input, int size, char *output)
}
template <typename T>
auto decode_rle(const T *input, int size, T *output, auto f_cmp_indicator) {
auto decode_rle(const T *input, int size, T *output) {
if (size < 2) {
return (uint8_t *)output;
}
@ -65,7 +65,8 @@ auto decode_rle(const T *input, int size, T *output, auto f_cmp_indicator) {
const auto rle_indicator = (uint8_t)*input++;
while (1) {
auto c = *input++;
if (f_cmp_indicator(c) != rle_indicator) {
//msvc bad warn. workaround vvvvvvvv
if ((sizeof(T) == 1 ? c : ((uint16_t)c >> 8)) != rle_indicator) {
*output++ = c;
} else {
uint32_t count = sizeof(T) == 1 ? *input++ : (c & 0xFF);
@ -83,11 +84,3 @@ auto decode_rle(const T *input, int size, T *output, auto f_cmp_indicator) {
}
}
}
auto decode_rle(const uint16_t *input, int size, uint16_t *output) {
return decode_rle(input, size, output, [](auto c){return c >> 8;});
}
auto decode_rle(const uint8_t *input, const int size, uint8_t *output) {
return decode_rle(input, size, output, [](auto c){return c;});
}

169
src/unpaker/pak.cpp
View file

@ -1,169 +0,0 @@
/*
* AIM 1 unpaker
* Copyright (C) 2015 lzwdgc
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "pak.h"
#include <algorithm>
#include <assert.h>
#include <stdio.h>
#include <iostream>
#include "decode.h"
#define FREAD(var) fread(&var, 1, sizeof(var), f)
void header::load(FILE *f)
{
FREAD(unk1);
FREAD(unk2);
FREAD(number_of_files);
FREAD(unk3);
FREAD(number_of_chunks);
FREAD(chunk_size);
FREAD(unk5);
}
void record::load(FILE *f)
{
char n[0x50];
FREAD(n);
name = n;
FREAD(pos);
FREAD(len);
}
void record::write(string name, const vector<char> &data) const
{
name += "\\" + string(this->name);
string dir = name.substr(0, name.rfind('\\'));
system(string("mkdir " + dir + " 2> nul").c_str());
FILE *f = fopen(name.c_str(), "wb");
if (!f)
return;
fwrite(data.data(), 1, data.size(), f);
fclose(f);
}
int record::read(pak *pak, void *output, int size)
{
int file_start_pos = offset + pos;
int segment = file_start_pos / pak->h.chunk_size;
offset += size;
pak->segments[segment].decompress(segment);
auto file_start_pos2 = file_start_pos - segment * pak->h.chunk_size;
auto size3 = size;
if (pak->h.chunk_size - file_start_pos2 < size)
size3 = pak->h.chunk_size - file_start_pos2;
memcpy(output, pak->segments[segment].decoded + file_start_pos2, size3);
auto size_diff = size - size3;
uint32_t diff = 0;
for (char *out = (char *)output + size3; size_diff > 0; out += diff)
{
segment++;
pak->segments[segment].decompress(segment);
diff = pak->h.chunk_size;
if (diff >= size_diff)
diff = size_diff;
memcpy(out, pak->segments[segment].decoded, 4 * (diff >> 2));
size_diff -= diff;
memcpy(
out + 4 * (diff >> 2),
pak->segments[segment].decoded + 4 * (diff >> 2),
diff & 3);
}
return size;
}
void segment::load_header(FILE *f)
{
FREAD(unk1);
FREAD(algorithms);
FREAD(offset);
}
void segment::load_segment()
{
auto f = file;
fseek(f, offset, SEEK_SET);
/*if (algorithms == 0)
{
std::cerr << "Something is wrong. Maybe you trying to open aim2 files?\n";
std::cerr << "They can be opened with SDK extractor.\n";
throw std::runtime_error("error");
}*/
FREAD(size1);
size2 = size1;
if (algorithms & DA_2)
{
FREAD(size2);
fread(&decoded[0], 1, size2, f);
}
else
{
fread(&encoded[0], 1, size1, f);
}
}
void segment::decompress(int segment_id)
{
load_segment();
if (algorithms & DA_2) {
decode_f2((char*)decoded, size2, (char*)encoded);
}
if (algorithms & RLE_2_bytes) {
decode_rle((uint16_t *)encoded, size1, (uint16_t *)decoded);
} else if (algorithms & RLE_1_byte) {
decode_rle((uint8_t *)encoded, size1, (uint8_t *)decoded);
}
if (algorithms == None) {
decoded = encoded;
}
}
void pak::load(FILE *f)
{
h.load(f);
encoded.resize(h.chunk_size * 4);
decoded.resize(h.chunk_size * 4);
int n = h.number_of_files;
while (n--)
{
record rec;
rec.load(f);
files[rec.name] = rec;
}
n = h.number_of_chunks;
while (n--)
{
segment t;
t.load_header(f);
t.file = f;
t.encoded = encoded.data();
t.decoded = decoded.data();
segments.push_back(t);
}
}

95
src/unpaker/pak.h
View file

@ -1,95 +0,0 @@
/*
* AIM 1 unpaker
* Copyright (C) 2015 lzwdgc
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#pragma once
#include <map>
#include <stdint.h>
#include <string>
#include <vector>
using namespace std;
struct header
{
uint32_t unk1;
uint16_t unk2;
uint16_t number_of_files;
uint16_t unk3;
uint32_t number_of_chunks;
int32_t chunk_size;
uint32_t unk5;
void load(FILE *f);
};
struct record
{
std::string name;
uint32_t pos;
uint32_t len;
//
vector<char> data;
int offset = 0;
void load(FILE *f);
void write(string name, const vector<char> &data) const;
int read(struct pak *pak, void *output, int size);
};
struct segment
{
enum decode_algorithm : uint32_t
{
None = 0x0,
RLE_2_bytes = 0x1,
RLE_1_byte = 0x2,
DA_1 = 0x4, // not used
DA_2 = 0x8,
};
uint32_t unk1; // some file offset? trash?
decode_algorithm algorithms;
uint32_t offset;
uint32_t size1;
uint32_t size2;
//
FILE *file = 0;
uint8_t* encoded;
uint8_t* decoded;
void load_header(FILE *f);
void load_segment();
void decompress(int segment_id);
};
struct pak
{
header h;
vector<segment> segments;
map<string, record> files;
//
vector<uint8_t> encoded;
vector<uint8_t> decoded;
void load(FILE *f);
};

206
src/unpaker/unpaker.cpp
View file

@ -1,6 +1,6 @@
/*
* AIM 1 unpaker
* Copyright (C) 2015 lzwdgc
* AIM unpaker (for AIM and AIM2 games, AIM:R)
* Copyright (C) 2023 lzwdgc
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -16,53 +16,181 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <filesystem>
#include <common.h>
#include <mmap.h>
#include <types.h>
#include <primitives/filesystem.h>
#include <primitives/sw/main.h>
#include <primitives/sw/settings.h>
#include <primitives/sw/cl.h>
#include <algorithm>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdio.h>
#include <stdint.h>
#include <string>
#include "pak.h"
#include <lzma.h>
#include <lzo/lzo1x.h>
namespace fs = std::filesystem;
#include "decode.h"
void unpak(string fn)
{
FILE *f = fopen(fn.c_str(), "rb");
if (!f)
return;
pak p;
p.load(f);
using namespace std;
auto unpack = [&](auto &file)
{
cout << "Unpacking " << file.name << "\n";
vector<char> buf(file.len);
file.read(&p, &buf[0], file.len);
file.write(fn + ".dir", buf);
};
void unpack_file(path fn) {
primitives::templates2::mmap_file<uint8_t> f{fn};
stream s{f};
pak p = s;
auto descs = s.span<file_description>(p.n_files);
auto segments = s.span<segment>(p.n_blocks);
std::vector<uint8_t> decoded;
decoded.resize((segments.size() + 1) * p.block_size * 4);
auto pp = decoded.data();
progress_bar pb{segments.size(), 50};
for (auto &&seg : segments) {
s.p = f.p + seg.offset;
uint32_t len = s;
auto m2 = [&]() {
enum decode_algorithm : uint32_t {
none = 0x0,
lzo = 0x1,
lzma = 0x2,
rlew = 0x4, // https://moddingwiki.shikadi.net/wiki/Id_Software_RLEW_compression
};
switch (seg.algorithm) {
case decode_algorithm::none: {
memcpy(pp, s.p, len);
pp += len;
break;
}
case decode_algorithm::lzo: {
size_t outsz;
// use lzo1x_decompress_safe?
auto r2 = lzo1x_decompress(s.p, len, pp, &outsz, 0);
if (r2 != LZO_E_OK) {
throw std::runtime_error{"lzo error"};
}
pp += outsz;
break;
}
case decode_algorithm::rlew: {
auto base = s.p;
uint16_t flag = s;
while (s.p < base + len) {
uint16_t w = s;
if ((w & 0xFF00) == (flag << 8)) {
uint16_t count = (uint8_t)w;
if (count == 0xFF) {
uint16_t w2 = s;
*(decltype(w2) *)pp = w2;
pp += sizeof(w2);
continue;
}
uint16_t w2 = s;
count += 3;
while (count--) {
*(decltype(w2)*)pp = w2;
pp += sizeof(w2);
}
} else {
*(decltype(w)*)pp = w;
pp += sizeof(w);
}
}
break;
}
case decode_algorithm::lzma: {
uint8_t flags = s;
for (auto &[n,f] : p.files)
unpack(f);
fclose(f);
lzma_stream strm{};
strm.next_in = s.p;
strm.avail_in = len;
strm.next_out = pp;
strm.avail_out = p.block_size;
auto r = lzma_lzip_decoder(&strm, 10'000'000, flags);
if (r != LZMA_OK) {
throw std::runtime_error{"lzma error"};
}
r = lzma_code(&strm, LZMA_RUN);
if (r != LZMA_STREAM_END) {
throw std::runtime_error{"lzma error"};
}
pp += strm.total_out;
break;
}
default:
throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)};
}
};
auto m1 = [&]() {
enum decode_algorithm : uint32_t {
None = 0x0,
RLE_2_bytes = 0x1,
RLE_1_byte = 0x2,
decode_algorithm_1 = 0x4, // not used
decode_algorithm_2 = 0x8,
};
auto in = s.p;
auto size1 = len;
std::vector<uint8_t> vec;
if (seg.algorithm & decode_algorithm_1) {
// if you see this, check in git history decode_f1()
throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)};
}
if (seg.algorithm & decode_algorithm_2) {
uint32_t size2 = s;
vec.resize(std::max(size2 * 4, p.block_size));
decode_f2((char *)s.p, size2, (char *)vec.data());
in = vec.data();
}
if (seg.algorithm & RLE_2_bytes) {
pp = decode_rle((uint16_t *)in, size1, (uint16_t *)pp);
} else if (seg.algorithm & RLE_1_byte) {
pp = decode_rle((uint8_t *)in, size1, (uint8_t *)pp);
}
if (seg.algorithm == None) {
memcpy(pp, s.p, size1);
}
};
if (p.magic == 0) {
m1();
} else {
m2();
}
pb.step();
}
std::cout << "\n";
auto dir = fn += ".dir";
fs::create_directories(dir);
for (auto &&d : descs) {
auto fn = dir / d.name;
fs::create_directories(fn.parent_path());
std::cout << "unpacking " << fn << "\n";
primitives::templates2::mmap_file<uint8_t> f{fn, primitives::templates2::mmap_file<uint8_t>::rw{}};
f.alloc_raw(d.size);
memcpy(f.p, decoded.data() + d.offset, d.size);
}
}
int main(int argc, char *argv[])
{
if (argc != 2)
{
cerr << "Usage: " << argv[0] << " <archive.pak or dir>" << "\n";
return 1;
}
fs::path p = argv[1];
int main(int argc, char *argv[]) {
cl::opt<path> p(cl::Positional, cl::desc("<pack file or dir>"), cl::Required);
cl::ParseCommandLineOptions(argc, argv);
if (fs::is_regular_file(p)) {
unpak(p.string());
unpack_file(p);
} else if (fs::is_directory(p)) {
for (auto &&d : fs::directory_iterator{p}) {
if (d.path().extension() == ".pak") {
std::cout << "processing: " << d.path() << "\n";
try {
unpak(d.path().string());
} catch (std::exception &e) {
std::cerr << e.what() << "\n";
}
auto files = enumerate_files_like(p, ".*\\.pak", false);
for (auto &f : files) {
std::cout << "processing: " << f << "\n";
try {
unpack_file(f);
} catch (std::exception &e) {
std::cerr << e.what() << "\n";
}
}
} else {

236
src/unpaker2/unpaker2.cpp
View file

@ -1,236 +0,0 @@
/*
* AIM unpaker2 (for AIM2 only at the moment)
* Copyright (C) 2023 lzwdgc
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <buffer.h>
#include <mmap.h>
#include <primitives/filesystem.h>
#include <primitives/sw/main.h>
#include <primitives/sw/settings.h>
#include <primitives/sw/cl.h>
#include <algorithm>
#include <fstream>
#include <iostream>
#include <sstream>
#include <stdio.h>
#include <stdint.h>
#include <string>
#include <lzma.h>
#include <lzo/lzo1x.h>
#include "../unpaker/decode.h"
using namespace std;
#pragma pack(push, 1)
struct pak {
uint32_t magic;
uint16_t unk0;
uint32_t n_files;
uint32_t n_blocks;
uint32_t block_size;
uint32_t unk1;
};
struct file_description {
const char name[0x50];
uint32_t offset;
uint32_t size;
};
struct segment {
// some file offset? trash? crc? m1 has zlib crc table (png)?
uint32_t unk1;
uint32_t algorithm;
uint32_t offset;
};
#pragma pack(pop)
struct progress_bar {
const size_t max_elements;
const int displaylen;
int displaycur{};
int i{};
void step() {
auto progress_bar_pos = std::round((double)++i / max_elements * displaylen);
for (int i = displaycur; i < progress_bar_pos; ++i) {
std::cout << "#";
}
displaycur = progress_bar_pos;
}
};
void unpack_file(path fn) {
primitives::templates2::mmap_file<uint8_t> f{fn};
stream s{f};
pak p = s;
auto descs = s.span<file_description>(p.n_files);
auto segments = s.span<segment>(p.n_blocks);
std::vector<uint8_t> decoded;
decoded.resize((segments.size() + 1) * p.block_size * 4);
auto pp = decoded.data();
progress_bar pb{segments.size(), 50};
for (auto &&seg : segments) {
s.p = f.p + seg.offset;
uint32_t len = s;
auto m2 = [&]() {
enum decode_algorithm : uint32_t {
none = 0x0,
lzo = 0x1,
lzma = 0x2,
rlew = 0x4, // https://moddingwiki.shikadi.net/wiki/Id_Software_RLEW_compression
};
switch (seg.algorithm) {
case decode_algorithm::none: {
memcpy(pp, s.p, len);
pp += len;
break;
}
case decode_algorithm::lzo: {
size_t outsz;
// use lzo1x_decompress_safe?
auto r2 = lzo1x_decompress(s.p, len, pp, &outsz, 0);
if (r2 != LZO_E_OK) {
throw std::runtime_error{"lzo error"};
}
pp += outsz;
break;
}
case decode_algorithm::rlew: {
auto base = s.p;
uint16_t flag = s;
while (s.p < base + len) {
uint16_t w = s;
if ((w & 0xFF00) == (flag << 8)) {
uint16_t count = (uint8_t)w;
if (count == 0xFF) {
uint16_t w2 = s;
*(decltype(w2) *)pp = w2;
pp += sizeof(w2);
continue;
}
uint16_t w2 = s;
count += 3;
while (count--) {
*(decltype(w2)*)pp = w2;
pp += sizeof(w2);
}
} else {
*(decltype(w)*)pp = w;
pp += sizeof(w);
}
}
break;
}
case decode_algorithm::lzma: {
uint8_t flags = s;
lzma_stream strm{};
strm.next_in = s.p;
strm.avail_in = len;
strm.next_out = pp;
strm.avail_out = p.block_size;
auto r = lzma_lzip_decoder(&strm, 10'000'000, flags);
if (r != LZMA_OK) {
throw std::runtime_error{"lzma error"};
}
r = lzma_code(&strm, LZMA_RUN);
if (r != LZMA_STREAM_END) {
throw std::runtime_error{"lzma error"};
}
pp += strm.total_out;
break;
}
default:
throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)};
}
};
auto m1 = [&]() {
enum decode_algorithm : uint32_t {
None = 0x0,
RLE_2_bytes = 0x1,
RLE_1_byte = 0x2,
decode_algorithm_1 = 0x4, // not used
decode_algorithm_2 = 0x8,
};
auto in = s.p;
auto size1 = len;
std::vector<uint8_t> vec;
if (seg.algorithm & decode_algorithm_1) {
// if you see this, check in git history decode_f1()
throw std::runtime_error{"compression unsupported: "s + std::to_string(seg.algorithm)};
}
if (seg.algorithm & decode_algorithm_2) {
uint32_t size2 = s;
vec.resize(std::max(size2 * 4, p.block_size));
decode_f2((char *)s.p, size2, (char *)vec.data());
in = vec.data();
}
if (seg.algorithm & RLE_2_bytes) {
pp = decode_rle((uint16_t *)in, size1, (uint16_t *)pp);
} else if (seg.algorithm & RLE_1_byte) {
pp = decode_rle((uint8_t *)in, size1, (uint8_t *)pp);
}
if (seg.algorithm == None) {
//decoded = encoded;
}
};
if (p.magic == 0) {
m1();
} else {
m2();
}
pb.step();
}
std::cout << "\n";
auto dir = fn += ".dir";
fs::create_directories(dir);
for (auto &&d : descs) {
auto fn = dir / d.name;
fs::create_directories(fn.parent_path());
std::cout << "unpacking " << fn << "\n";
primitives::templates2::mmap_file<uint8_t> f{fn, primitives::templates2::mmap_file<uint8_t>::rw{}};
f.alloc_raw(d.size);
memcpy(f.p, decoded.data() + d.offset, d.size);
}
}
int main(int argc, char *argv[]) {
cl::opt<path> p(cl::Positional, cl::desc("<pack file or dir>"), cl::Required);
cl::ParseCommandLineOptions(argc, argv);
if (fs::is_regular_file(p)) {
unpack_file(p);
} else if (fs::is_directory(p)) {
auto files = enumerate_files_like(p, ".*\\.pak", false);
for (auto &f : files) {
std::cout << "processing: " << f << "\n";
try {
unpack_file(f);
} catch (std::exception &e) {
std::cerr << e.what() << "\n";
}
}
} else {
throw std::runtime_error("Bad fs object");
}
return 0;
}

3
sw.cpp
View file

@ -53,8 +53,7 @@ void build(Solution &s)
add_exe_with_common("tm_converter");
add_exe("name_generator");
add_exe_with_common("save_loader");
add_exe("unpaker");
add_exe_with_common("unpaker2") +=
add_exe_with_common("unpaker") +=
"org.sw.demo.oberhumer.lzo.lzo"_dep,
"org.sw.demo.xz_utils.lzma"_dep
;