From 6e7a55f42227b5e062cfdc7469d010f439918203 Mon Sep 17 00:00:00 2001 From: d0k3 Date: Wed, 30 Nov 2016 21:01:05 +0100 Subject: [PATCH] Enable browsing the RomFS dir --- source/common.h | 2 +- source/game/game.h | 1 + source/game/romfs.c | 88 ++++++++++++++++++ source/game/romfs.h | 79 +++++++++++++++++ source/virtual/vgame.c | 186 +++++++++++++++++++++++++++++++++++---- source/virtual/vgame.h | 3 + source/virtual/virtual.c | 15 +++- 7 files changed, 355 insertions(+), 19 deletions(-) create mode 100644 source/game/romfs.c create mode 100644 source/game/romfs.h diff --git a/source/common.h b/source/common.h index a722076..b21395f 100644 --- a/source/common.h +++ b/source/common.h @@ -59,7 +59,7 @@ #define GAME_BUFFER_SIZE (0x100000) // buffer area defines (in use by vgame.c) #define VGAME_BUFFER ((u8*)0x21600000) -#define VGAME_BUFFER_SIZE (0x100000) +#define VGAME_BUFFER_SIZE (0x200000) // 2MB, big RomFS // buffer area defines (in use by image.c, for RAMdrive) #define RAMDRV_BUFFER_O3DS ((u8*)0x22200000) // in O3DS FCRAM #define RAMDRV_SIZE_O3DS (0x01C00000) // 28MB diff --git a/source/game/game.h b/source/game/game.h index 2f46fcc..8b87e59 100644 --- a/source/game/game.h +++ b/source/game/game.h @@ -5,3 +5,4 @@ #include "ncsd.h" #include "ncch.h" #include "exefs.h" +#include "romfs.h" diff --git a/source/game/romfs.c b/source/game/romfs.c new file mode 100644 index 0000000..d7091cb --- /dev/null +++ b/source/game/romfs.c @@ -0,0 +1,88 @@ +#include "romfs.h" + +// validate header by checking offsets and sizes +u32 ValidateLv3Header(RomFsLv3Header* lv3, u32 max_size) { + return ((lv3->size_header == 0x28) && + (lv3->offset_dirhash >= lv3->size_header) && + (lv3->offset_dirmeta >= lv3->offset_dirhash + lv3->size_dirhash) && + (lv3->offset_filehash >= lv3->offset_dirmeta + lv3->size_dirmeta) && + (lv3->offset_filemeta >= lv3->offset_filehash + lv3->size_filehash) && + (lv3->offset_filedata >= lv3->offset_filemeta + lv3->size_filemeta) && + (!max_size || (lv3->offset_filedata <= max_size))) ? 0 : 1; +} + +// build index of RomFS lvl3 +u32 BuildLv3Index(RomFsLv3Index* index, u8* lv3) { + RomFsLv3Header* hdr = (RomFsLv3Header*) lv3; + index->header = hdr; + index->dirhash = (u32*) (void*) (lv3 + hdr->offset_dirhash); + index->dirmeta = lv3 + hdr->offset_dirmeta; + index->filehash = (u32*) (void*) (lv3 + hdr->offset_filehash); + index->filemeta = lv3 + hdr->offset_filemeta; + index->filedata = NULL; + + index->mod_dir = (hdr->size_dirhash / sizeof(u32)); + index->mod_file = (hdr->size_filehash / sizeof(u32)); + index->size_dirmeta = hdr->size_dirmeta; + index->size_filemeta = hdr->size_filemeta; + + return 0; +} + +// hash lvl3 path - this is used to find the first offset in the file / dir hash table +u32 HashLv3Path(u16* wname, u32 name_len, u32 offset_parent) { + u32 hash = offset_parent ^ 123456789; + for (u32 i = 0; i < name_len; i++) + hash = ((hash>>5) | (hash<<27)) ^ wname[i]; + return hash; +} + +RomFsLv3DirMeta* GetLv3DirMeta(const char* name, u32 offset_parent, RomFsLv3Index* index) { + RomFsLv3DirMeta* meta; + + // wide name + u16 wname[256]; + u32 name_len = strnlen(name, 256); + for (name_len = 0; name[name_len]; name_len++) + wname[name_len] = name[name_len]; // poor mans UTF-8 -> UTF-16 + + // hashing, first offset + u32 hash = HashLv3Path(wname, name_len, offset_parent); + u32 offset = index->dirhash[hash % index->mod_dir]; + // process the hashbucket (make sure we got the correct data) + // slim chance of endless loop with broken lvl3 here + for (; offset < index->size_dirmeta; offset = meta->offset_samehash) { + meta = (RomFsLv3DirMeta*) (index->dirmeta + offset); + if ((offset_parent == meta->offset_parent) && + (name_len == meta->name_len / 2) && + (memcmp(wname, meta->wname, name_len * 2) == 0)) + break; + } + + return (offset >= index->size_dirmeta) ? NULL : meta; +} + +RomFsLv3FileMeta* GetLv3FileMeta(const char* name, u32 offset_parent, RomFsLv3Index* index) { + RomFsLv3FileMeta* meta; + + // wide name + u16 wname[256]; + u32 name_len = strnlen(name, 256); + for (name_len = 0; name[name_len]; name_len++) + wname[name_len] = name[name_len]; // poor mans UTF-8 -> UTF-16 + + // hashing, first offset + u32 hash = HashLv3Path(wname, name_len, offset_parent); + u32 offset = index->filehash[hash % index->mod_file]; + // process the hashbucket (make sure we got the correct data) + // slim chance of endless loop with broken lvl3 here + for (; offset < index->size_filemeta; offset = meta->offset_samehash) { + meta = (RomFsLv3FileMeta*) (index->filemeta + offset); + if ((offset_parent == meta->offset_parent) && + (name_len == meta->name_len / 2) && + (memcmp(wname, meta->wname, name_len * 2) == 0)) + break; + } + + return (offset >= index->size_filemeta) ? NULL : meta; +} diff --git a/source/game/romfs.h b/source/game/romfs.h new file mode 100644 index 0000000..666341c --- /dev/null +++ b/source/game/romfs.h @@ -0,0 +1,79 @@ +#pragma once + +#include "common.h" + +#define ROMFS_MAGIC 0x49, 0x56, 0x46, 0x43, 0x00, 0x00, 0x01, 0x00 +#define OFFSET_LV3 0x1000 + +#define LV3_GET_DIR(offset, idx) \ + ((RomFsLv3DirMeta*) (void*) ((idx)->dirmeta + (offset))) +#define LV3_GET_FILE(offset, idx) \ + ((RomFsLv3FileMeta*) (void*) ((idx)->filemeta + (offset))) +#define LV3_GET_SIBLING_DIR(dm, idx) \ + ((RomFsLv3DirMeta*) (void*) (((dm)->offset_sibling < (idx)->size_dirmeta) ?\ + ((idx)->dirmeta + (dm)->offset_sibling : NULL))) +#define LV3_GET_SIBLING_FILE(fm, idx) \ + ((RomFsLv3FileMeta*) (void*) (((fm)->offset_sibling < (idx)->size_filemeta) ?\ + ((idx)->filemeta + (fm(->offset_sibling : NULL))) +#define LV3_GET_PARENT_DIR(dfm, idx) \ + ((RomFsLv3DirMeta*) (void*) (((dfm)->offset_parent < (idx)->size_dirmeta) ?\ + ((idx)->dirmeta + (dfm)->offset_parent : NULL))) +#define LV3_GET_CHILD_FILE(dm, idx) \ + ((RomFsLv3FileMeta*) (void*) (((dm)->offset_file < (idx)->size_filemeta) ?\ + ((idx)->filemeta + (dm)->offset_file : NULL))) +#define LV3_GET_CHILD_DIR(dm, idx) \ + ((RomFsLv3DirMeta*) (void*) (((dm)->offset_child < (idx)->size_dirmeta) ?\ + ((idx)->dirmeta + (dm)->offset_parent : NULL))) + + +typedef struct { + u32 size_header; + u32 offset_dirhash; + u32 size_dirhash; + u32 offset_dirmeta; + u32 size_dirmeta; + u32 offset_filehash; + u32 size_filehash; + u32 offset_filemeta; + u32 size_filemeta; + u32 offset_filedata; +} __attribute__((packed)) RomFsLv3Header; + +typedef struct { + u32 offset_parent; + u32 offset_sibling; + u32 offset_child; + u32 offset_file; + u32 offset_samehash; + u32 name_len; + u16 wname[256]; // 256 assumed to be max name length +} __attribute__((packed)) RomFsLv3DirMeta; + +typedef struct { + u32 offset_parent; + u32 offset_sibling; + u64 offset_data; + u64 size_data; + u32 offset_samehash; + u32 name_len; + u16 wname[256]; // 256 assumed to be max name length +} __attribute__((packed)) RomFsLv3FileMeta; + +typedef struct { + RomFsLv3Header* header; + u32* dirhash; + u8* dirmeta; + u32* filehash; + u8* filemeta; + u8* filedata; + u32 mod_dir; + u32 mod_file; + u32 size_dirmeta; + u32 size_filemeta; +} __attribute__((packed)) RomFsLv3Index; + +u32 ValidateLv3Header(RomFsLv3Header* lv3, u32 max_size); +u32 BuildLv3Index(RomFsLv3Index* index, u8* lv3); +u32 HashLv3Path(u16* wname, u32 name_len, u32 offset_parent); +RomFsLv3DirMeta* GetLv3DirMeta(const char* name, u32 offset_parent, RomFsLv3Index* index); +RomFsLv3FileMeta* GetLv3FileMeta(const char* name, u32 offset_parent, RomFsLv3Index* index); diff --git a/source/virtual/vgame.c b/source/virtual/vgame.c index ed093db..39070ee 100644 --- a/source/virtual/vgame.c +++ b/source/virtual/vgame.c @@ -4,19 +4,21 @@ #include "aes.h" #include "ff.h" -#define VFLAG_EXTHDR (1<<26) -#define VFLAG_CIA (1<<27) // unused, see below -#define VFLAG_NCSD (1<<28) // unused, see below -#define VFLAG_NCCH (1<<29) -#define VFLAG_EXEFS (1<<30) -#define VFLAG_ROMFS (1<<31) +#define VFLAG_EXTHDR (1<<25) +#define VFLAG_CIA (1<<26) // unused, see below +#define VFLAG_NCSD (1<<27) // unused, see below +#define VFLAG_NCCH (1<<28) +#define VFLAG_EXEFS (1<<29) +#define VFLAG_ROMFS (1<<30) +#define VFLAG_LV3 (1<<31) #define VDIR_CIA VFLAG_CIA #define VDIR_NCSD VFLAG_NCSD #define VDIR_NCCH VFLAG_NCCH #define VDIR_EXEFS VFLAG_EXEFS #define VDIR_ROMFS VFLAG_ROMFS -#define VDIR_GAME (VDIR_CIA|VDIR_NCSD|VDIR_NCCH|VDIR_EXEFS|VDIR_ROMFS) +#define VDIR_LV3 VFLAG_LV3 +#define VDIR_GAME (VDIR_CIA|VDIR_NCSD|VDIR_NCCH|VDIR_EXEFS|VDIR_ROMFS|VDIR_LV3) #define MAX_N_TEMPLATES 2048 // this leaves us with enough room (128kB reserved) @@ -52,10 +54,10 @@ static u32 vgame_type = 0; static u32 base_vdir = 0; -static VirtualFile* templates_cia = (VirtualFile*) VGAME_BUFFER; // first 116kb reserved (enough for ~2000 entries) -static VirtualFile* templates_ncsd = (VirtualFile*) VGAME_BUFFER + 0x1D000; // 4kb reserved (enough for ~80 entries) -static VirtualFile* templates_ncch = (VirtualFile*) VGAME_BUFFER + 0x1E000; // 4kb reserved (enough for ~80 entries) -static VirtualFile* templates_exefs = (VirtualFile*) VGAME_BUFFER + 0x1F000; // 4kb reserved (enough for ~80 entries) +static VirtualFile* templates_cia = (VirtualFile*) VGAME_BUFFER; // first 52kb reserved (enough for 950 entries) +static VirtualFile* templates_ncsd = (VirtualFile*) VGAME_BUFFER + 0xE800; // 2kb reserved (enough for 36 entries) +static VirtualFile* templates_ncch = (VirtualFile*) VGAME_BUFFER + 0xF000; // 2kb reserved (enough for 36 entries) +static VirtualFile* templates_exefs = (VirtualFile*) VGAME_BUFFER + 0xF800; // 2kb reserved (enough for 36 entries) static int n_templates_cia = -1; static int n_templates_ncsd = -1; static int n_templates_ncch = -1; @@ -66,11 +68,15 @@ static u64 offset_ncsd = (u64) -1; static u64 offset_ncch = (u64) -1; static u64 offset_exefs = (u64) -1; static u64 offset_romfs = (u64) -1; +static u64 offset_lv3 = (u64) -1; +static u64 offset_lv3fd = (u64) -1; -static CiaStub* cia = (CiaStub*) (VGAME_BUFFER + 0xF4000); // 48kB reserved - should be enough by far -static NcsdHeader* ncsd = (NcsdHeader*) (VGAME_BUFFER + 0xF3000); // 512 byte reserved -static NcchHeader* ncch = (NcchHeader*) (VGAME_BUFFER + 0xF3200); // 512 byte reserved -static ExeFsHeader* exefs = (ExeFsHeader*) (VGAME_BUFFER + 0xF3400); // 512 byte reserved +static CiaStub* cia = (CiaStub*) (VGAME_BUFFER + 0x10000); // 62.5kB reserved - should be enough by far +static NcsdHeader* ncsd = (NcsdHeader*) (VGAME_BUFFER + 0x1FA00); // 512 byte reserved +static NcchHeader* ncch = (NcchHeader*) (VGAME_BUFFER + 0x1FC00); // 512 byte reserved +static ExeFsHeader* exefs = (ExeFsHeader*) (VGAME_BUFFER + 0x1FE00); // 512 byte reserved +static u8* romfslv3 = (u8*) (VGAME_BUFFER + 0x20000); // 1920kB reserved +static RomFsLv3Index lv3idx; bool BuildVGameExeFsDir(void) { VirtualFile* templates = templates_exefs; @@ -157,6 +163,12 @@ bool BuildVGameNcchDir(void) { templates[n].keyslot = 0xFF; // crypto ? templates[n].flags = VFLAG_ROMFS; n++; + if (!NCCH_ENCRYPTED(ncch)) { + memcpy(templates + n, templates + n - 1, sizeof(VirtualFile)); + strncpy(templates[n].name, NAME_NCCH_ROMFSDIR, 32); + templates[n].flags |= VFLAG_DIR; + n++; + } } n_templates_ncch = n; @@ -328,6 +340,8 @@ u32 InitVGameDrive(void) { // prerequisite: game file mounted as image offset_ncch = (u64) -1; offset_exefs = (u64) -1; offset_romfs = (u64) -1; + offset_lv3 = (u64) -1; + offset_lv3fd = (u64) -1; base_vdir = (type == GAME_CIA) ? VDIR_CIA : (type == GAME_NCSD) ? VDIR_NCSD : (type == GAME_NCCH) ? VDIR_NCCH : 0; if (!base_vdir) return 0; @@ -385,11 +399,112 @@ bool OpenVGameDir(VirtualDir* vdir, VirtualFile* ventry) { return false; offset_exefs = vdir->offset; if (!BuildVGameExeFsDir()) return false; + } else if ((vdir->flags & VDIR_ROMFS) && (offset_romfs != vdir->offset)) { + // validate romFS magic + u8 magic[] = { ROMFS_MAGIC }; + u8 header[sizeof(magic)]; + if ((ReadImageBytes(header, vdir->offset, sizeof(magic)) != 0) || + (memcmp(magic, header, sizeof(magic)) != 0)) + return false; + // validate lv3 header + RomFsLv3Header* lv3 = (RomFsLv3Header*) romfslv3; + for (u32 i = 1; i < 8; i++) { + offset_lv3 = vdir->offset + (i*OFFSET_LV3); + if (ReadImageBytes(romfslv3, offset_lv3, sizeof(RomFsLv3Header)) != 0) + return false; + if (ValidateLv3Header(lv3, VGAME_BUFFER_SIZE - 0x20000) == 0) + break; + offset_lv3 = (u64) -1; + } + if ((offset_lv3 == (u64) -1) || (ReadImageBytes(romfslv3, offset_lv3, lv3->offset_filedata) != 0)) + return false; + offset_lv3fd = offset_lv3 + lv3->offset_filedata; + offset_romfs = vdir->offset; + BuildLv3Index(&lv3idx, romfslv3); + } + + // for romfs dir: switch to lv3 dir object + if (vdir->flags & VDIR_ROMFS) { + vdir->index = -1; + vdir->offset = 0; + vdir->size = 0; + vdir->flags &= ~VDIR_ROMFS; + vdir->flags |= VDIR_LV3; } return true; // error (should not happen) } +bool ReadVGameDirLv3(VirtualFile* vfile, VirtualDir* vdir) { + BuildLv3Index(&lv3idx, romfslv3); + vfile->flags = VFLAG_LV3; + + // start from parent dir object + if (vdir->index == -1) vdir->index = 0; + + // first child dir object, skip if not available + if (vdir->index == 0) { + RomFsLv3DirMeta* parent = LV3_GET_DIR(vdir->offset, &lv3idx); + if (!parent) return false; + if (parent->offset_child != (u32) -1) { + vdir->offset = (u64) parent->offset_child; + vdir->index = 1; + vfile->flags |= VFLAG_DIR; + vfile->offset = vdir->offset; + return true; + } else vdir->index = 2; + } + + // parse sibling dirs + if (vdir->index == 1) { + RomFsLv3DirMeta* current = LV3_GET_DIR(vdir->offset, &lv3idx); + if (!current) return false; + if (current->offset_sibling != (u32) -1) { + vdir->offset = (u64) current->offset_sibling; + vfile->flags |= VFLAG_DIR; + vfile->offset = vdir->offset; + return true; + } else if (current->offset_parent != (u32) -1) { + vdir->offset = (u64) current->offset_parent; + vdir->index = 2; + } else return false; + } + + // first child file object, skip if not available + if (vdir->index == 2) { + RomFsLv3DirMeta* parent = LV3_GET_DIR(vdir->offset, &lv3idx); + if (!parent) return false; + if (parent->offset_file != (u32) -1) { + vdir->offset = (u64) parent->offset_file; + vdir->index = 3; + RomFsLv3FileMeta* lv3file = LV3_GET_FILE(vdir->offset, &lv3idx); + if (!lv3file) return false; + vfile->offset = vdir->offset; + vfile->size = lv3file->size_data; + return true; + } else vdir->index = 4; + } + + // parse sibling files + if (vdir->index == 3) { + RomFsLv3FileMeta* current = LV3_GET_FILE(vdir->offset, &lv3idx); + if (!current) return false; + if (current->offset_sibling != (u32) -1) { + vdir->offset = current->offset_sibling; + RomFsLv3FileMeta* lv3file = LV3_GET_FILE(vdir->offset, &lv3idx); + if (!lv3file) return false; + vfile->offset = vdir->offset; + vfile->size = lv3file->size_data; + return true; + } else if (current->offset_parent != (u32) -1) { + vdir->offset = current->offset_parent; + vdir->index = 4; + } else return false; + } + + return false; +} + bool ReadVGameDir(VirtualFile* vfile, VirtualDir* vdir) { VirtualFile* templates; int n = 0; @@ -406,6 +521,8 @@ bool ReadVGameDir(VirtualFile* vfile, VirtualDir* vdir) { } else if (vdir->flags & VDIR_EXEFS) { templates = templates_exefs; n = n_templates_exefs; + } else if (vdir->flags & VDIR_LV3) { + return ReadVGameDirLv3(vfile, vdir); } if (++vdir->index < n) { @@ -419,6 +536,12 @@ bool ReadVGameDir(VirtualFile* vfile, VirtualDir* vdir) { int ReadVGameFile(const VirtualFile* vfile, u8* buffer, u32 offset, u32 count) { u32 vfoffset = vfile->offset; + if (vfile->flags & VFLAG_LV3) { + RomFsLv3FileMeta* lv3file; + if (vfile->flags & VFLAG_DIR) return -1; + lv3file = LV3_GET_FILE(vfile->offset, &lv3idx); + vfoffset = offset_lv3fd + lv3file->offset_data; + } int ret = ReadImageBytes(buffer, vfoffset + offset, count); if (ret != 0) return ret; /*if ((ret != 0) && (vfile->keyslot <= 0x40)) { // crypto @@ -434,3 +557,36 @@ int ReadVGameFile(const VirtualFile* vfile, u8* buffer, u32 offset, u32 count) { }*/ return 0; } + +bool GetVGameFilename(char* name, const VirtualFile* vfile, u32 n_chars) { + if (!(vfile->flags & VFLAG_LV3)) { + snprintf(name, n_chars, "%s", vfile->name); + return true; + } + + u16* wname = NULL; + u32 name_len = 0; + + if (vfile->flags & VFLAG_DIR) { + RomFsLv3DirMeta* dirmeta = LV3_GET_DIR(vfile->offset, &lv3idx); + if (!dirmeta) return false; + wname = dirmeta->wname; + name_len = dirmeta->name_len / 2; + } else { + RomFsLv3FileMeta* filemeta = LV3_GET_FILE(vfile->offset, &lv3idx); + if (!filemeta) return false; + wname = filemeta->wname; + name_len = filemeta->name_len / 2; + } + memset(name, 0, n_chars); + for (u32 i = 0; (i < (n_chars-1)) && (i < name_len); i++) + name[i] = wname[i]; // poor mans UTF-16 -> UTF-8 + + return true; +} + +bool MatchVGameFilename(const char* name, const VirtualFile* vfile, u32 n_chars) { + char vg_name[256]; + if (!GetVGameFilename(vg_name, vfile, 256)) return false; + return (strncasecmp(name, vg_name, n_chars) == 0); +} diff --git a/source/virtual/vgame.h b/source/virtual/vgame.h index 300e49d..dae0d26 100644 --- a/source/virtual/vgame.h +++ b/source/virtual/vgame.h @@ -11,3 +11,6 @@ bool OpenVGameDir(VirtualDir* vdir, VirtualFile* ventry); bool ReadVGameDir(VirtualFile* vfile, VirtualDir* vdir); int ReadVGameFile(const VirtualFile* vfile, u8* buffer, u32 offset, u32 count); // int WriteVGameFile(const VirtualFile* vfile, const u8* buffer, u32 offset, u32 count); // writing is not enabled + +bool GetVGameFilename(char* name, const VirtualFile* vfile, u32 n_chars); +bool MatchVGameFilename(const char* name, const VirtualFile* vfile, u32 n_chars); diff --git a/source/virtual/virtual.c b/source/virtual/virtual.c index 9971a2d..579d35a 100644 --- a/source/virtual/virtual.c +++ b/source/virtual/virtual.c @@ -90,7 +90,9 @@ bool GetVirtualFile(VirtualFile* vfile, const char* path) { for (name = strtok(lpath + 3, "/"); name && vdir.virtual_src; name = strtok(NULL, "/")) { while (true) { if (!ReadVirtualDir(vfile, &vdir)) return false; - if (strncasecmp(name, vfile->name, 32) == 0) + if ((virtual_src != VRT_GAME) && (strncasecmp(name, vfile->name, 32) == 0)) + break; // entry found + if ((virtual_src == VRT_GAME) && MatchVGameFilename(name, vfile, 256)) break; // entry found } if (!OpenVirtualDir(&vdir, vfile)) @@ -115,8 +117,15 @@ bool GetVirtualDirContents(DirStruct* contents, const char* path, const char* pa return false; // get dir reader object while ((contents->n_entries < MAX_DIR_ENTRIES) && (ReadVirtualDir(&vfile, &vdir))) { DirEntry* entry = &(contents->entry[contents->n_entries]); - if (pattern && !MatchName(pattern, vfile.name)) continue; - snprintf(entry->path, 256, "%s/%s", path, vfile.name); + if (!(vfile.flags & VRT_GAME)) { + if (pattern && !MatchName(pattern, vfile.name)) continue; + snprintf(entry->path, 256, "%s/%s", path, vfile.name); + } else { + char name[256]; + if (!GetVGameFilename(name, &vfile, 256)) return false; + if (pattern && !MatchName(pattern, name)) continue; + snprintf(entry->path, 256, "%s/%s", path, name); + } entry->name = entry->path + strnlen(path, 256) + 1; entry->size = vfile.size; entry->type = (vfile.flags & VFLAG_DIR) ? T_DIR : T_FILE;