From 141b388b9211eb37cf5cb3e197830a25a01d09fc Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Fri, 4 Sep 2026 18:30:20 -0500 Subject: [PATCH] More build update progress. Windows now building. --- CHANGELOG | 5 + CMakeLists.txt | 31 +++++- CMakePresets.json | 2 +- INSTALL | 43 ++++++-- build-all.sh | 37 ++++--- cmake/Superbuild.cmake | 19 +++- cmake/linux-x86_64.cmake | 26 +++++ cmake/zig/x86_64-windows-gnu.cmake | 34 ++++++ cmake/zig/zig.cmake | 69 ++++++++++++ src/singe.c | 171 ++++++++++++++++++++++++++++- src/vfs.c | 153 +++++++++++++++++++++++++- src/vfs.h | 4 + 12 files changed, 559 insertions(+), 35 deletions(-) create mode 100644 cmake/linux-x86_64.cmake create mode 100644 cmake/zig/x86_64-windows-gnu.cmake create mode 100644 cmake/zig/zig.cmake diff --git a/CHANGELOG b/CHANGELOG index 56ffd213f..3f202d677 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -10,6 +10,11 @@ API Changes - The name now expands to "SINGE Is Not a Game Emulator", in the banner, the manual, INSTALL, and the Windows file description. +- lfs.dir, lfs.attributes and lfs.symlinkattributes see inside a packed + game (the union of its loose directory, its data overlay and the + database), and lfs.mkdir and lfs.rmdir act on its data overlay, so + frameworks that list or create their own directories work packed. + - A game can be one file: singe --pack DIRECTORY GAME.game writes the game's scripts, art, sounds, fonts, and video into an SQLite database, and singe GAME.game (or the menu, which lists every .game file beside the game diff --git a/CMakeLists.txt b/CMakeLists.txt index ff5621094..391dc4d72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -405,8 +405,23 @@ add_executable(${CMAKE_PROJECT_NAME} ${MANYMOUSE_SOURCE} ) if(WIN32) - enable_language(RC) - target_sources(${CMAKE_PROJECT_NAME} PRIVATE ${GENERATED_DIR}/singe.rc ${GENERATED_DIR}/icon.ico) + if(SINGE_ZIG_TARGET) + # CMake archives every object into objects.a for MinGW links, and zig's linker cannot take a + # compiled resource from inside an archive; compiled on its own it links as a plain input. + add_custom_command( + OUTPUT ${GENERATED_DIR}/singe.res + COMMAND ${CMAKE_RC_COMPILER} /fo ${GENERATED_DIR}/singe.res ${GENERATED_DIR}/singe.rc + DEPENDS ${GENERATED_DIR}/singe.rc ${GENERATED_DIR}/icon.ico + COMMENT "Compiling resources" + VERBATIM + ) + add_custom_target(singeResources DEPENDS ${GENERATED_DIR}/singe.res) + add_dependencies(${CMAKE_PROJECT_NAME} singeResources) + target_link_options(${CMAKE_PROJECT_NAME} PRIVATE ${GENERATED_DIR}/singe.res) + else() + enable_language(RC) + target_sources(${CMAKE_PROJECT_NAME} PRIVATE ${GENERATED_DIR}/singe.rc ${GENERATED_DIR}/icon.ico) + endif() endif() # Output name matches the release artifact: Singe-v3.00-Linux-x86_64 @@ -417,6 +432,8 @@ set_target_properties(${CMAKE_PROJECT_NAME} PROPERTIES OUTPUT_NAME "Singe-v${PRO # Warnings apply to our code only; the vendored libraries are not ours to fix. set_source_files_properties(${SINGE_SOURCE} PROPERTIES COMPILE_OPTIONS "-Wall;-Wextra") +# librs232 stamps its build date into the module; clang under zig makes that an error by default. +set_source_files_properties(thirdparty/librs232/bindings/lua/luars232.c PROPERTIES COMPILE_OPTIONS "-Wno-date-time") target_compile_definitions(${CMAKE_PROJECT_NAME} PRIVATE RS232_STATIC @@ -484,8 +501,14 @@ elseif(KANGAROO_OS STREQUAL "macos") message(FATAL_ERROR "OSXCROSS_TARGET_DIR must be set for macOS builds.") endif() elseif(KANGAROO_OS STREQUAL "windows") + # A GUI-subsystem executable, so no console window opens; zig's linker wants the flag spelled out. + if(SINGE_ZIG_TARGET) + set(windowsSubsystem -Wl,--subsystem,windows) + else() + set(windowsSubsystem -mwindows) + endif() set(SYSTEM_LIBS - -mwindows + ${windowsSubsystem} -static -lmingw32 -latomic @@ -565,7 +588,7 @@ set(STATIC_LIBS if(NOT KANGAROO_OS STREQUAL "macos") list(APPEND STATIC_LIBS ${BUILD_DIR}/lib/libjpeg.a - ${BUILD_DIR}/lib/libpng.a + ${BUILD_DIR}/lib/libpng16.a ) endif() diff --git a/CMakePresets.json b/CMakePresets.json index 89c54865a..0c30f1ecc 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -20,7 +20,7 @@ "displayName": "Linux x86_64", "inherits": "base", "binaryDir": "${sourceDir}/.builddir/linux/x86_64/superbuild", - "toolchainFile": "${sourceDir}/../toolchains/cmake/x86_64-linux-gnu.cmake", + "toolchainFile": "${sourceDir}/cmake/linux-x86_64.cmake", "cacheVariables": { "SINGE_TRIPLE": "x86_64-linux-gnu", "SINGE_CROSS_OS": "linux", diff --git a/INSTALL b/INSTALL index faf01664a..af3eeea35 100644 --- a/INSTALL +++ b/INSTALL @@ -48,17 +48,40 @@ is extracted on first run. BUILDING FROM SOURCE ==================== -The build is a CMake superbuild: one preset per platform builds every -vendored library (zlib, zstd, SDL3 and its satellites, OpenSSL, FFmpeg) -into .builddir// and then Singe against them. The cross -toolchains live in ../toolchains (see toolchains.sh there); build-all.sh -prepares that environment and runs the preset: +On a Debian based system a checkout builds with one command: - ./build-all.sh every supported platform - ./build-all.sh linux x86_64 one platform - ./build-all.sh linux x86_64 --target singe Singe alone, libraries as built + ./build-all.sh linux x86_64 + +It installs the host packages it needs with apt (asks for your sudo +password once), then runs the CMake superbuild, which builds every vendored +library (zlib, zstd, SDL3 and its satellites, OpenSSL, FFmpeg) into +.builddir// and then Singe against them. The finished binary is +copied to .builddir/Singe-v--. + +Host packages (what build-all.sh installs): build-essential cmake +pkg-config perl nasm imagemagick lua5.4 ffmpeg asciidoctor +ruby-asciidoctor-pdf autoconf automake libtool libasound-dev libxi-dev +libvdpau-dev libva-dev libdrm-dev libgl-dev libx11-dev. + +Cross builds need nothing else installed: + + ./build-all.sh windows x86_64 + +fetches a pinned zig release (version and SHA-256 in cmake/zig/zig.cmake) +into .builddir/toolchains on first use and uses it as the cross compiler; +zig carries the Windows headers and import libraries itself. The Pi and +macOS presets are not yet self-contained; they still expect the toolchains +repository beside this one (see PLAN.md section 17 for the plan to move +them to zig too). + +Useful forms (anything after the platform goes to cmake --build): + + ./build-all.sh linux x86_64 --target singe Singe alone ./build-all.sh linux x86_64 --target rebuild-ffmpeg redo one library -The finished binary is copied to .builddir/Singe-v--. -Presets are linux-x86_64, pi-aarch64, macos-aarch64 and windows-x86_64 +Without the wrapper: + + cmake --preset linux-x86_64 && cmake --build --preset linux-x86_64 + +Presets: linux-x86_64, windows-x86_64, pi-aarch64, macos-aarch64 (CMakePresets.json). diff --git a/build-all.sh b/build-all.sh index 496b47075..a29120ed9 100755 --- a/build-all.sh +++ b/build-all.sh @@ -22,7 +22,7 @@ # Builds Singe and every vendored library for one platform, or all of them. The work is done # by the CMake superbuild (cmake/Superbuild.cmake, one preset per platform in -# CMakePresets.json); this script only prepares the toolchain environment and calls it: +# CMakePresets.json); this script installs the host packages and calls it: # # ./build-all.sh every supported platform # ./build-all.sh linux x86_64 one platform @@ -39,12 +39,18 @@ function buildAll() { local ARCH=$2 shift 2 - # Activate the toolchain for this platform (PATH, compilers, sysroot, osxcross variables). - source <(../toolchains/toolchains.sh use ${ARCH} ${OS}) - - # The Pi sysroot needs the development packages the build links against. - if [[ "${OS}" == "pi" ]]; then - sudo chroot ${SYSROOT} apt-get -y install libasound-dev libxi-dev libvdpau-dev libdrm-dev + # Linux builds with the host compiler and Windows with the zig the build fetches itself. The Pi + # and macOS presets still lean on the toolchains repository beside this one until they move to + # zig as well (PLAN.md section 17). + if [[ "${OS}" == "pi" || "${OS}" == "macos" ]]; then + if [[ ! -x ../toolchains/toolchains.sh ]]; then + echo "error: ${OS} still needs ../toolchains/toolchains.sh" >&2 + exit 1 + fi + source <(../toolchains/toolchains.sh use ${ARCH} ${OS}) + if [[ "${OS}" == "pi" ]]; then + sudo chroot ${SYSROOT} apt-get -y install libasound-dev libxi-dev libvdpau-dev libdrm-dev + fi fi cmake --preset ${OS}-${ARCH} @@ -57,17 +63,16 @@ function buildAll() { # -o pipefail = stop pipeline if any step fails set -euo pipefail -if [[ ! -x ../toolchains/toolchains.sh ]]; then - echo "error: ../toolchains/toolchains.sh not found (see INSTALL)" >&2 - exit 1 -fi - - mkdir -p ${G_BUILDDIR} -# These are required for the build. +# Host packages the build needs on a Debian based system (the same list is in INSTALL). sudo apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + perl \ + nasm \ imagemagick \ lua5.4 \ ffmpeg \ @@ -80,7 +85,9 @@ sudo apt-get install -y \ libxi-dev \ libvdpau-dev \ libva-dev \ - libdrm-dev + libdrm-dev \ + libgl-dev \ + libx11-dev if [[ $# -ge 2 ]]; then diff --git a/cmake/Superbuild.cmake b/cmake/Superbuild.cmake index 681044868..4889cdd72 100644 --- a/cmake/Superbuild.cmake +++ b/cmake/Superbuild.cmake @@ -146,13 +146,13 @@ function(singeRebuildTarget name) endfunction() -# ===== zlib (configure script; its own include directory, not the prefix, as before) ===== +# ===== zlib (configure script; the generated zconf.h lives in the build directory) ===== set(zlibBinary ${SB_PREFIX}/build/zlib) ExternalProject_Add(zlib SOURCE_DIR ${SB_THIRDPARTY}/zlib BINARY_DIR ${zlibBinary} - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env "CC=${SB_CC}" "CFLAGS=-I${SB_THIRDPARTY}/zlib" ${SB_THIRDPARTY}/zlib/configure --prefix=${SB_PREFIX} --static + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E env "CC=${SB_CC}" "AR=${CMAKE_AR}" "RANLIB=${SB_RANLIB}" "CFLAGS=-I${zlibBinary} ${SB_CFLAGS_BASE}" ${SB_THIRDPARTY}/zlib/configure --prefix=${SB_PREFIX} --static BUILD_COMMAND ${SB_ENV} make INSTALL_COMMAND ${SB_ENV} make install LOG_CONFIGURE ON LOG_BUILD ON LOG_INSTALL ON LOG_OUTPUT_ON_FAILURE ON @@ -182,6 +182,12 @@ singeCmakeProject(SDL3_mixer ${SB_THIRDPARTY}/SDL3_mixer "SDL3" singeCmakeProject(SDL3_ttf ${SB_THIRDPARTY}/SDL3_ttf "SDL3" "-DBUILD_SHARED_LIBS=off;-DSDLTTF_VENDORED=on;-DSDLTTF_HARFBUZZ=off;-DSDLTTF_PLUTOSVG=off;-DSDLTTF_SAMPLES=off;${sdl3Dir}" "${SB_ENV}") +# SDL3_ttf builds its vendored FreeType but does not install it, and Singe links it directly. +ExternalProject_Add_Step(SDL3_ttf freetype + COMMAND ${CMAKE_COMMAND} -E copy ${SB_PREFIX}/build/SDL3_ttf/external/freetype/libfreetype.a ${SB_PREFIX}/lib/libfreetype.a + DEPENDEES install + COMMENT "Installing libfreetype.a" +) # ===== OpenSSL (its own Configure; the target name follows the platform) ===== @@ -207,6 +213,13 @@ elseif(KANGAROO_OS STREQUAL "macos") else() set(opensslTarget "") endif() +# OpenSSL's generated assembly uses GNU as directives clang's assembler rejects; under zig the +# portable C paths serve, which is all LuaSec's traffic needs. +if(SINGE_ZIG_TARGET) + set(opensslAsm no-asm) +else() + set(opensslAsm "") +endif() # OpenSSL's Configure derives the source tree from the build directory's position and only # copes with a build directory nested inside its own tree, so this one lives there. set(opensslBinary ${SB_THIRDPARTY}/openssl/.builddir) @@ -214,7 +227,7 @@ ExternalProject_Add(openssl SOURCE_DIR ${SB_THIRDPARTY}/openssl BINARY_DIR ${opensslBinary} DEPENDS zlib zstd - CONFIGURE_COMMAND ${SB_ENV} perl ../Configure ${opensslTarget} --prefix=${SB_PREFIX} --libdir=lib --with-zlib-include=${SB_PREFIX}/include --with-zlib-lib=${SB_PREFIX}/lib --with-zstd-include=${SB_PREFIX}/include --with-zstd-lib=${SB_PREFIX}/lib no-apps no-docs no-dso no-dynamic-engine no-module no-shared no-tests zlib enable-zstd + CONFIGURE_COMMAND ${SB_ENV} perl ../Configure ${opensslTarget} --prefix=${SB_PREFIX} --libdir=lib --with-zlib-include=${SB_PREFIX}/include --with-zlib-lib=${SB_PREFIX}/lib --with-zstd-include=${SB_PREFIX}/include --with-zstd-lib=${SB_PREFIX}/lib ${opensslAsm} no-apps no-docs no-dso no-dynamic-engine no-module no-shared no-tests zlib enable-zstd BUILD_COMMAND ${SB_ENV} make build_sw INSTALL_COMMAND ${SB_ENV} make install_sw LOG_CONFIGURE ON LOG_BUILD ON LOG_INSTALL ON LOG_OUTPUT_ON_FAILURE ON diff --git a/cmake/linux-x86_64.cmake b/cmake/linux-x86_64.cmake new file mode 100644 index 000000000..c4396dbfe --- /dev/null +++ b/cmake/linux-x86_64.cmake @@ -0,0 +1,26 @@ +# +# Singe 3 +# Copyright (C) 2006-2026 Scott Duensing +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public Licens +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + +# Linux x86_64 built on a Linux x86_64 host with its own gcc and development packages. + +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(KANGAROO_OS linux) +set(KANGAROO_ARCH x86_64) diff --git a/cmake/zig/x86_64-windows-gnu.cmake b/cmake/zig/x86_64-windows-gnu.cmake new file mode 100644 index 000000000..85ebe869e --- /dev/null +++ b/cmake/zig/x86_64-windows-gnu.cmake @@ -0,0 +1,34 @@ +# +# Singe 3 +# Copyright (C) 2006-2026 Scott Duensing +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public Licens +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + +# Windows x86_64 with zig as the cross compiler: zig ships the mingw-w64 headers and import +# libraries, so nothing else has to be installed. + +include(${CMAKE_CURRENT_LIST_DIR}/zig.cmake) + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +singeZigToolchain(x86_64-windows-gnu) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) +set(KANGAROO_OS windows) +set(KANGAROO_ARCH x86_64) diff --git a/cmake/zig/zig.cmake b/cmake/zig/zig.cmake new file mode 100644 index 000000000..acb879356 --- /dev/null +++ b/cmake/zig/zig.cmake @@ -0,0 +1,69 @@ +# +# Singe 3 +# Copyright (C) 2006-2026 Scott Duensing +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public Licens +# as published by the Free Software Foundation; either version 3 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301, USA. +# + +# The one place the zig version lives. Toolchain files include this and name a target; the +# compiler, archiver and resource compiler are small wrapper scripts so every tool that wants a +# single program name (FFmpeg's configure, OpenSSL's, CMake itself) gets one. + +# Pinned release, fetched into .builddir/toolchains on first use and checked against its hash. +set(ZIG_VERSION 0.15.2) +set(ZIG_SHA256 02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239) +set(ZIG_NAME zig-x86_64-linux-${ZIG_VERSION}) +set(ZIG_URL https://ziglang.org/download/${ZIG_VERSION}/${ZIG_NAME}.tar.xz) +get_filename_component(ZIG_TOOLCHAINS ${CMAKE_CURRENT_LIST_DIR}/../../.builddir/toolchains ABSOLUTE) +set(ZIG_ROOT ${ZIG_TOOLCHAINS}/${ZIG_NAME}) +set(ZIG_EXECUTABLE ${ZIG_ROOT}/zig) +if(NOT EXISTS ${ZIG_EXECUTABLE}) + message(STATUS "Downloading zig ${ZIG_VERSION} from ${ZIG_URL}") + file(MAKE_DIRECTORY ${ZIG_TOOLCHAINS}) + file(DOWNLOAD ${ZIG_URL} ${ZIG_TOOLCHAINS}/${ZIG_NAME}.tar.xz EXPECTED_HASH SHA256=${ZIG_SHA256} STATUS status) + list(GET status 0 code) + if(NOT code EQUAL 0) + message(FATAL_ERROR "zig download failed: ${status}") + endif() + file(ARCHIVE_EXTRACT INPUT ${ZIG_TOOLCHAINS}/${ZIG_NAME}.tar.xz DESTINATION ${ZIG_TOOLCHAINS}) + file(REMOVE ${ZIG_TOOLCHAINS}/${ZIG_NAME}.tar.xz) + if(NOT EXISTS ${ZIG_EXECUTABLE}) + message(FATAL_ERROR "zig ${ZIG_VERSION} did not unpack to ${ZIG_ROOT}.") + endif() +endif() + +# Writes the wrapper scripts for a target into the build tree and points CMake at them. +function(singeZigToolchain target) + set(dir ${CMAKE_BINARY_DIR}/zig-${target}) + file(MAKE_DIRECTORY ${dir}) + # The compiler wrappers drop one flag: FFmpeg's configure adds binutils' --pic-executable ASLR + # workaround to every test link on Windows, and zig's linker driver refuses it. They also add + set(filter "for a in \"$@\"; do [[ $a == -Wl,--pic-executable* ]] || args+=(\"$a\"); done") + # -fno-sanitize=undefined: zig cc instruments for UBSan by default and expects its runtime at + # link time; a release build wants neither. + foreach(tool "cc;-target ${target} -fno-sanitize=undefined" "c++;-target ${target} -fno-sanitize=undefined" "ar;" "ranlib;" "rc;") + list(GET tool 0 name) + list(GET tool 1 args) + file(WRITE ${dir}/zig-${name} "#!/bin/bash\nargs=()\n${filter}\nexec \"${ZIG_EXECUTABLE}\" ${name} ${args} \"$\{args[@]}\"\n") + file(CHMOD ${dir}/zig-${name} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + endforeach() + set(CMAKE_C_COMPILER ${dir}/zig-cc CACHE FILEPATH "C compiler" FORCE) + set(CMAKE_CXX_COMPILER ${dir}/zig-c++ CACHE FILEPATH "C++ compiler" FORCE) + set(CMAKE_AR ${dir}/zig-ar CACHE FILEPATH "archiver" FORCE) + set(CMAKE_RANLIB ${dir}/zig-ranlib CACHE FILEPATH "ranlib" FORCE) + set(CMAKE_RC_COMPILER ${dir}/zig-rc CACHE FILEPATH "resource compiler" FORCE) + set(SINGE_ZIG_TARGET ${target} CACHE STRING "zig target triple" FORCE) +endfunction() diff --git a/src/singe.c b/src/singe.c index f1293554c..ce3cb23d0 100644 --- a/src/singe.c +++ b/src/singe.c @@ -49,6 +49,8 @@ LUASOCKET_API int luaopen_socket_serial(lua_State *L); int luaopen_luars232(lua_State *L); // Nor for lsqlite3. int luaopen_lsqlite3(lua_State *L); +// The module table below needs this before the prototypes. +static int32_t _luaopenLfs(lua_State *L); // There is no header for ssl.config binding. Make our own. LSEC_API int luaopen_ssl_config(lua_State *L); @@ -347,7 +349,7 @@ static MIX_Track *_effectTracks[EFFECT_TRACKS]; // Lua Modules static const LuaModuleT _luaModules[] = { // LuaFileSystem - MODC("lfs", luaopen_lfs), + MODC("lfs", _luaopenLfs), // SQLite for script data MODC("sqlite3", luaopen_lsqlite3), // LuaSocket @@ -455,6 +457,11 @@ static void _fontDestroy(FontT *font); static void _freezeGame(bool freeze); static void _heldListUpdate(HeldKeyT *list, int32_t *count, bool down, int32_t keysym, int32_t scancode); static void _installFileHooks(lua_State *L); +static int32_t _lfsAttributes(lua_State *L); +static int32_t _lfsDir(lua_State *L); +static int32_t _lfsDirIterator(lua_State *L); +static int32_t _lfsMkdir(lua_State *L); +static int32_t _lfsRmdir(lua_State *L); static int32_t _loadAudioCalibration(void); static void _loadControlsFile(const char *path); static SDL_Surface *_loadEmbeddedPng(const unsigned char *data, unsigned int length); @@ -466,6 +473,7 @@ static char *_luaFormat(lua_State *L, const char *method, const char *fmt static int32_t _luaIoHook(lua_State *L); static int32_t _luaLoadfile(lua_State *L); static int32_t _luaLoadFile(lua_State *L, const char *name, const char *mode); +static int32_t _luaopenLfs(lua_State *L); static int32_t _luaPanic(lua_State *L); static int32_t _luaSearcher(lua_State *L); static void _luaTrace(lua_State *L, const char *method, const char *fmt, ...) __attribute__((format(printf, 3, 4))); @@ -1323,6 +1331,144 @@ static void _installFileHooks(lua_State *L) { } +// lfs.attributes through the vfs: a packed entry answers with mode, size and modification; a +// plain filesystem path goes to the original (upvalue 1). +static int32_t _lfsAttributes(lua_State *L) { + const char *name = luaL_checkstring(L, 1); + const char *attribute = luaL_optstring(L, 2, NULL); + int64_t size = 0; + int64_t modified = 0; + bool directory = false; + + if (vfsIsFilesystem(name)) { + lua_pushvalue(L, lua_upvalueindex(1)); + lua_insert(L, 1); + lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); + return lua_gettop(L); + } + directory = vfsIsDirectory(name); + if (!directory && !vfsStat(name, &size, &modified)) { + lua_pushnil(L); + lua_pushfstring(L, "cannot obtain information from file '%s'", name); + return 2; + } + lua_newtable(L); + lua_pushstring(L, directory ? "directory" : "file"); + lua_setfield(L, -2, "mode"); + lua_pushinteger(L, (lua_Integer)size); + lua_setfield(L, -2, "size"); + lua_pushinteger(L, (lua_Integer)modified); + lua_setfield(L, -2, "modification"); + lua_pushinteger(L, (lua_Integer)modified); + lua_setfield(L, -2, "change"); + lua_pushinteger(L, (lua_Integer)modified); + lua_setfield(L, -2, "access"); + if (attribute != NULL) { + lua_getfield(L, -1, attribute); + } + + return 1; +} + + +// lfs.dir through the vfs: a packed directory iterates the union of loose, overlay and packed +// entries; a plain filesystem path goes to the original (upvalue 1). +static int32_t _lfsDir(lua_State *L) { + const char *name = luaL_checkstring(L, 1); + char **list = NULL; + int32_t count = 0; + int32_t i = 0; + + if (vfsIsFilesystem(name)) { + lua_pushvalue(L, lua_upvalueindex(1)); + lua_insert(L, 1); + lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); + return lua_gettop(L); + } + if (!vfsIsDirectory(name)) { + return luaL_error(L, "cannot open %s: No such file or directory", name); + } + list = vfsList(name, &count); + lua_createtable(L, count, 0); + for (i = 0; i < count; i++) { + lua_pushstring(L, list[i]); + lua_rawseti(L, -2, i + 1); + } + vfsListFree(list, count); + lua_pushinteger(L, 0); + lua_pushcclosure(L, _lfsDirIterator, 2); + + return 1; +} + + +// The iterator lfs.dir returns for a packed directory: upvalues are the entry table and the index. +static int32_t _lfsDirIterator(lua_State *L) { + lua_Integer index = lua_tointeger(L, lua_upvalueindex(2)) + 1; + + lua_pushinteger(L, index); + lua_copy(L, -1, lua_upvalueindex(2)); + lua_pop(L, 1); + lua_rawgeti(L, lua_upvalueindex(1), index); + + return 1; +} + + +// lfs.mkdir through the vfs: inside a packed game the directory is made where its files will be +// written, the data overlay; a plain filesystem path goes to the original (upvalue 1). +static int32_t _lfsMkdir(lua_State *L) { + const char *name = luaL_checkstring(L, 1); + char *path = NULL; + bool ok = false; + + if (vfsIsFilesystem(name)) { + lua_pushvalue(L, lua_upvalueindex(1)); + lua_insert(L, 1); + lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); + return lua_gettop(L); + } + path = vfsFilePath(name, true); + ok = utilMkDirP(path, 0755); + free(path); + if (!ok) { + lua_pushnil(L); + lua_pushfstring(L, "cannot create %s", name); + return 2; + } + lua_pushboolean(L, true); + + return 1; +} + + +// lfs.rmdir through the vfs: removes the overlay directory of a packed name; the packed copy +// itself is read only and stays. +static int32_t _lfsRmdir(lua_State *L) { + const char *name = luaL_checkstring(L, 1); + char *path = NULL; + bool ok = false; + + if (vfsIsFilesystem(name)) { + lua_pushvalue(L, lua_upvalueindex(1)); + lua_insert(L, 1); + lua_call(L, lua_gettop(L) - 1, LUA_MULTRET); + return lua_gettop(L); + } + path = vfsFilePath(name, true); + ok = (rmdir(path) == 0); + free(path); + if (!ok) { + lua_pushnil(L); + lua_pushfstring(L, "cannot remove %s", name); + return 2; + } + lua_pushboolean(L, true); + + return 1; +} + + // The per-machine audio delay lives beside the data directories, since it is not a property of any game. static int32_t _loadAudioCalibration(void) { char *path = utilCreateString("%s%s", _global.conf->dataDirBase, AUDIO_CALIBRATION_FILE); @@ -1561,6 +1707,29 @@ static int32_t _luaLoadFile(lua_State *L, const char *name, const char *mode) { return status; } +// require("lfs") with dir and attributes routed through the vfs, so a packed game can list itself. +static int32_t _luaopenLfs(lua_State *L) { + luaopen_lfs(L); + lua_getfield(L, -1, "dir"); + lua_pushcclosure(L, _lfsDir, 1); + lua_setfield(L, -2, "dir"); + lua_getfield(L, -1, "attributes"); + lua_pushcclosure(L, _lfsAttributes, 1); + lua_setfield(L, -2, "attributes"); + lua_getfield(L, -1, "symlinkattributes"); + lua_pushcclosure(L, _lfsAttributes, 1); + lua_setfield(L, -2, "symlinkattributes"); + lua_getfield(L, -1, "mkdir"); + lua_pushcclosure(L, _lfsMkdir, 1); + lua_setfield(L, -2, "mkdir"); + lua_getfield(L, -1, "rmdir"); + lua_pushcclosure(L, _lfsRmdir, 1); + lua_setfield(L, -2, "rmdir"); + + return 1; +} + + static int32_t _luaPanic(lua_State *L) { lua_Debug ar; int32_t level = 0; diff --git a/src/vfs.c b/src/vfs.c index c05a62d26..9a85a2f7b 100644 --- a/src/vfs.c +++ b/src/vfs.c @@ -33,6 +33,7 @@ */ #include +#include #include #include #include @@ -71,6 +72,7 @@ typedef struct DatabaseS { sqlite3 *db; sqlite3_stmt *assetStmt; // size, data by path sqlite3_stmt *chunkStmt; // data by path and chunk number + sqlite3_stmt *listStmt; // paths under a prefix, for directory listings struct DatabaseS *next; } DatabaseT; @@ -97,10 +99,13 @@ struct VfsStreamS { static bool _assetExists(DatabaseT *db, const char *key); +static bool _assetDirectory(DatabaseT *db, const char *key); static bool _assetSize(DatabaseT *db, const char *key, int64_t *size); static DatabaseT *_databaseOpen(const char *path); static bool _databaseReadMeta(DatabaseT *db, const char *key, char **value); static bool _fileModified(const char *path, int64_t *size, int64_t *modified); +static void _listAdd(char ***list, int32_t *count, const char *name); +static void _listDirectory(const char *path, char ***list, int32_t *count); static bool _isAbsolute(const char *name); static bool _isEngineName(const char *name); static char *_normalise(const char *name); @@ -125,6 +130,24 @@ static bool _assetExists(DatabaseT *db, const char *key) { } +// True when any asset lives below the key, which is what a directory is in a database. +static bool _assetDirectory(DatabaseT *db, const char *key) { + char *from = utilCreateString("%s/", key); + char *to = utilCreateString("%s0", key); + bool found = false; + + sqlite3_reset(db->listStmt); + sqlite3_bind_text(db->listStmt, 1, from, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(db->listStmt, 2, to, -1, SQLITE_TRANSIENT); + found = (sqlite3_step(db->listStmt) == SQLITE_ROW); + sqlite3_reset(db->listStmt); + free(from); + free(to); + + return found; +} + + static bool _assetSize(DatabaseT *db, const char *key, int64_t *size) { bool found = false; @@ -196,6 +219,10 @@ static DatabaseT *_databaseOpen(const char *path) { if (sqlite3_prepare_v2(db->db, "SELECT data FROM chunks WHERE path = ? AND chunk = ?", -1, &db->chunkStmt, NULL) != SQLITE_OK) { utilDie("%s has no chunks table: %s", path, sqlite3_errmsg(db->db)); } + // Everything below a directory key sorts between "key/" and "key0" ('0' follows '/'). + if (sqlite3_prepare_v2(db->db, "SELECT path FROM assets WHERE path >= ? AND path < ? ORDER BY path", -1, &db->listStmt, NULL) != SQLITE_OK) { + utilDie("%s: %s", path, sqlite3_errmsg(db->db)); + } db->chunkBytes = VFS_CHUNK_BYTES; if (_databaseReadMeta(db, META_CHUNK, &value)) { db->chunkBytes = strtoll(value, NULL, 10); @@ -261,9 +288,15 @@ static bool _isEngineName(const char *name) { if ((utilStricmp(name, ENGINE_DIRECTORY) == 0) || ((strncmp(name, ENGINE_DIRECTORY, length) == 0) && (name[length] == '/'))) { return true; } + // The data directory base, when it is a real prefix: "data/" or an absolute path. With no + // --datadir the base is "./", which normalises to nothing and must not match everything. if (_dataDirBase != NULL) { base = _normalise(_dataDirBase); - engine = (strncmp(name, base, strlen(base)) == 0); + length = strlen(base); + while ((length > 0) && (base[length - 1] == '/')) { + length--; + } + engine = (length > 0) && (strncmp(name, base, length) == 0) && ((name[length] == '/') || (name[length] == 0)); free(base); } @@ -271,6 +304,38 @@ static bool _isEngineName(const char *name) { } +// Appends a name to a listing unless it is already there (case-insensitively, like the keys). +static void _listAdd(char ***list, int32_t *count, const char *name) { + int32_t i = 0; + + for (i = 0; i < *count; i++) { + if (utilStricmp((*list)[i], name) == 0) { + return; + } + } + *list = (char **)realloc(*list, (size_t)(*count + 1) * sizeof(char *)); + (*list)[*count] = strdup(name); + (*count)++; +} + + +// Adds the entries of a filesystem directory to a listing, if it exists. +static void _listDirectory(const char *path, char ***list, int32_t *count) { + DIR *dir = opendir(path); + struct dirent *de = NULL; + + if (dir == NULL) { + return; + } + while ((de = readdir(dir)) != NULL) { + if ((strcmp(de->d_name, ".") != 0) && (strcmp(de->d_name, "..") != 0)) { + _listAdd(list, count, de->d_name); + } + } + closedir(dir); +} + + // Forward slashes, no leading "./", no doubled slashes. Case is untouched. static char *_normalise(const char *name) { char *out = strdup(name); @@ -573,6 +638,91 @@ bool vfsIsDatabase(const char *path) { } +// A directory, loose or packed: a packed one exists when any asset sits below the key. +bool vfsIsDirectory(const char *name) { + TargetT target; + struct stat info; + bool found = false; + + if (!_resolve(name, &target)) { + return false; + } + found = (stat(target.path, &info) == 0) && S_ISDIR(info.st_mode); + if (!found && (target.db != NULL)) { + found = ((stat(target.overlay, &info) == 0) && S_ISDIR(info.st_mode)) || _assetDirectory(target.db, target.key); + } + _targetFree(&target); + + return found; +} + + +// True when the name is a plain filesystem path, so a caller may use the filesystem directly. +bool vfsIsFilesystem(const char *name) { + TargetT target; + bool plain = true; + + if (_resolve(name, &target)) { + plain = (target.db == NULL); + _targetFree(&target); + } + + return plain; +} + + +// The entries directly under a directory name, from the loose directory, the overlay and the +// database together, each name once. Free with vfsListFree. +char **vfsList(const char *name, int32_t *count) { + TargetT target; + char **list = NULL; + char *from = NULL; + char *to = NULL; + char *entry = NULL; + const char *path = NULL; + const char *slash = NULL; + size_t prefix = 0; + + *count = 0; + if (!_resolve(name, &target)) { + return NULL; + } + _listDirectory(target.path, &list, count); + if (target.db != NULL) { + _listDirectory(target.overlay, &list, count); + prefix = strlen(target.key); + from = (prefix == 0) ? strdup("") : utilCreateString("%s/", target.key); + to = (prefix == 0) ? strdup("\x7f") : utilCreateString("%s0", target.key); + sqlite3_reset(target.db->listStmt); + sqlite3_bind_text(target.db->listStmt, 1, from, -1, SQLITE_TRANSIENT); + sqlite3_bind_text(target.db->listStmt, 2, to, -1, SQLITE_TRANSIENT); + while (sqlite3_step(target.db->listStmt) == SQLITE_ROW) { + path = (const char *)sqlite3_column_text(target.db->listStmt, 0) + strlen(from); + slash = strchr(path, '/'); + entry = slash ? utilStrndup(path, (size_t)(slash - path)) : strdup(path); + _listAdd(&list, count, entry); + free(entry); + } + sqlite3_reset(target.db->listStmt); + free(from); + free(to); + } + _targetFree(&target); + + return list; +} + + +void vfsListFree(char **list, int32_t count) { + int32_t i = 0; + + for (i = 0; i < count; i++) { + free(list[i]); + } + free(list); +} + + // Read-only stream for the SDL loaders. Packed assets are served from memory. SDL_IOStream *vfsOpenIO(const char *name) { TargetT target; @@ -614,6 +764,7 @@ void vfsQuit(void) { next = db->next; sqlite3_finalize(db->assetStmt); sqlite3_finalize(db->chunkStmt); + sqlite3_finalize(db->listStmt); sqlite3_close(db->db); free(db->path); free(db->loose); diff --git a/src/vfs.h b/src/vfs.h index 85facde0c..5aee084f7 100644 --- a/src/vfs.h +++ b/src/vfs.h @@ -48,6 +48,10 @@ typedef struct VfsStreamS VfsStreamT; bool vfsExists(const char *name); +bool vfsIsDirectory(const char *name); +bool vfsIsFilesystem(const char *name); +char **vfsList(const char *name, int32_t *count); +void vfsListFree(char **list, int32_t count); char *vfsFilePath(const char *name, bool forWriting); void vfsInit(const char *container, const char *dataDirBase, const char *dataDir); bool vfsIsDatabase(const char *path);