From cc6eb18bafb92b0a1a4ee3ad8403d34bf32e01eb Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Thu, 4 May 2023 20:19:09 -0500 Subject: [PATCH] Build archive being unpacked into project folder. And then it crashes. --- CMakeLists.txt | 4 + include/ssh.h | 1 + include/utils.h | 30 +++++ src/main.c | 2 + src/messages.c | 2 +- src/project.c | 60 +++++++-- src/ssh.c | 29 ++++ src/utils.c | 190 ++++++++++++++++++++++++++- thirdparty/bzip2-bzip2-1.0.8.tar.bz2 | 3 + thirdparty/libarchive-3.6.2.tar.gz | 3 + tools/prebuild.sh | 22 ++++ 11 files changed, 332 insertions(+), 14 deletions(-) create mode 100644 thirdparty/bzip2-bzip2-1.0.8.tar.bz2 create mode 100644 thirdparty/libarchive-3.6.2.tar.gz diff --git a/CMakeLists.txt b/CMakeLists.txt index 669a5ad..5292331 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,8 @@ add_custom_target(GENERATE_UI_HEADERS ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libtcc.a ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/tcc/libtcc1.a ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libsigsegv.a + ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libbz2.a + ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libarchive.a ) add_dependencies(${CMAKE_PROJECT_NAME} GENERATE_UI_HEADERS) @@ -125,6 +127,8 @@ target_link_libraries(${CMAKE_PROJECT_NAME} ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libtcc.a ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/tcc/libtcc1.a ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libsigsegv.a + ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libarchive.a + ${CMAKE_SOURCE_DIR}/thirdparty-installed/lib/libbz2.a ${GTK3_LIBRARIES} -ldl -pthread diff --git a/include/ssh.h b/include/ssh.h index 0d760b5..3ee5d6b 100644 --- a/include/ssh.h +++ b/include/ssh.h @@ -71,6 +71,7 @@ typedef struct SFTPS { SSHT *sshConnect(char *hostname, uint16_t port, char *user, char *password); void sshDisconnect(SSHT **sshData); +gboolean sshSFTPDelete(SSHT *sshData, char *target); //int sshExecute(SSHT *sshData, char *command, char **output); gboolean sshSFTPFileExists(SSHT *sshData, char *target); SFTPT *sshSFTPReceive(SSHT *sshData, char *remote, char *local, SFTPCallback callback); diff --git a/include/utils.h b/include/utils.h index 6806a64..630212d 100644 --- a/include/utils.h +++ b/include/utils.h @@ -25,6 +25,8 @@ #include "common.h" +#include "archive.h" +#include "archive_entry.h" #ifdef _WIN32 @@ -34,11 +36,37 @@ #endif +struct ArchiveS; + +typedef void (*archiveCallback)(struct ArchiveS*); + +typedef struct ArchiveS { + struct archive *in; // Compressed input file. + struct archive *out; // Decompressed output files. + struct archive_entry *entry; // Current archive entry. + int result; // Result of various archive functions. + int compression; // Compression percentage. + int fileCount; // Current file number being processed. + uint64_t compressed; // Compressed bytes processed. + uint64_t uncompressed; // Uncompressed bytes processed. + size_t size; // Size of compressed file. + char *archive; // Name of compressed file. + char *outPath; // Path to store uncompressed files. + char *currentFile; // Name of current file being processed. + gboolean copying; // Are we copying bytes? + gboolean finished; // Decompress complete? + gboolean success; // Did it succeed? + void *userData; // Anything the user needs to keep. + archiveCallback callback; // Non-Null, call when complete/failed. +} ArchiveT; + + extern char __utilFilenameBuffer[FILENAME_MAX]; char *utilCreateString(char *format, ...); char *utilCreateStringVArgs(char *format, va_list args); +ArchiveT *utilDecompress(char *archive, char *outPath, archiveCallback callback, void *userData); char *utilDeobfuscateASCII(char *obfuscated); void utilDequote(char *string); void utilEnsureBufferSize(unsigned char **buffer, int *length, int wanted); @@ -62,6 +90,8 @@ gboolean utilMkDirP(const char *dir, const mode_t mode); char *utilObfuscateASCII(char *clearText); gboolean utilQuestionDialog(GtkWidget *parent, char *title, char *question); void utilSetDirty(WindowDataT *self, gboolean dirty); +void utilShutdown(void); +void utilStartup(void); void utilUpdatePath(WindowDataT *self); void utilWindowRegister(gpointer windowData); int utilWindowsCloseAll(void); diff --git a/src/main.c b/src/main.c index 3179e07..e51fa11 100644 --- a/src/main.c +++ b/src/main.c @@ -46,6 +46,7 @@ int main(int argc, char **argv) { #endif gtk_init(&argc, &argv); + utilStartup(); // Make sure we have a config & resources folder. __resourcePath = utilCreateString("%s%cjoeydev%cresources%c", g_get_user_config_dir(), UTIL_PATH_CHAR, UTIL_PATH_CHAR, UTIL_PATH_CHAR); @@ -78,6 +79,7 @@ int main(int argc, char **argv) { scintilla_release_resources(); sshShutdown(); httpShutdown(); + utilShutdown(); DEL(__resourcePath); diff --git a/src/messages.c b/src/messages.c index 66c7d94..50c09b3 100644 --- a/src/messages.c +++ b/src/messages.c @@ -104,7 +104,7 @@ void message(MessageTypesT level, char *format, ...) { // Force paint. gtk_widget_show_all(row); utilForceUpdate(); - _autoScroll = 5; // Try 5 times to show the new row. + _autoScroll = 15; // Try several times to show the new row. tok = strtok(NULL, "\n"); } diff --git a/src/project.c b/src/project.c index 04cb9c9..3923c3f 100644 --- a/src/project.c +++ b/src/project.c @@ -36,6 +36,10 @@ #include "editor.h" +#define LOCAL_BUILD_RESULTS "build.tar.bz2" +#define REMOTE_BUILD_RESULTS "build/build.tar.bz2" + + enum ProjectColumnsE { COL_FILENAME = 0, COL_COUNT @@ -116,6 +120,7 @@ EVENT void btnEditRawClicked(GtkButton *widget, gpointer userData); EVENT void btnEditRecipeClicked(GtkButton *widget, gpointer userData); EVENT void btnNewRecipeClicked(GtkButton *widget, gpointer userData); static void clearRecipeData(ProjectDataT *self); +static void decompressBuild(ArchiveT *archive); static void dialogCookOptions(char *filename, ProjectDataT *self); static int findRecipeData(ProjectDataT *self, char *key); static void loadConfig(ProjectDataT *self); @@ -239,6 +244,19 @@ static void clearRecipeData(ProjectDataT *self) { } +static void decompressBuild(ArchiveT *archive) { + ProjectDataT *self = (ProjectDataT *)archive->userData; + char *temp; + + // Delete archive. + temp = utilCreateString("%s%s", self->windowData.path, LOCAL_BUILD_RESULTS); + unlink(temp); + DEL(temp); + + //***TODO*** Process build results. +} + + static void dialogCookOptions(char *filename, ProjectDataT *self) { GtkWidget *dialogCookSettings; GtkWidget *lblRaw; @@ -968,7 +986,7 @@ EVENT void menuProjectBuildBuild(GtkWidget *object, gpointer userData) { } // Do we have a build package to clean up? - if (sshSFTPFileExists(ssh, "build/build.tar.bz2")) { + if (sshSFTPFileExists(ssh, REMOTE_BUILD_RESULTS)) { message(MSG_INFO, "Removing stale build results"); } @@ -1097,11 +1115,17 @@ gboolean projectAddToTree(ProjectDataT *self, char *filename) { static void receiveSFTP(SFTPT *sftp) { ProjectDataT *self = (ProjectDataT *)sftp->userData; + char *temp; if (sftp->finished) { if (sftp->success) { - //***TODO*** Unpack results + message(MSG_INFO, "Cleaning up remote build"); + sshSFTPDelete(sftp->sshData, REMOTE_BUILD_RESULTS); + //***TODO*** Delete any local "results" folder. message(MSG_INFO, "Unpacking build"); + temp = utilCreateString("%s%s", self->windowData.path, LOCAL_BUILD_RESULTS); + utilDecompress(temp, self->windowData.path, decompressBuild, self); + DEL(temp); } else { message(MSG_ERROR, "Receiving %s from build server - FAILED", sftp->remoteName); message(MSG_INFO, "Build canceled"); @@ -1279,6 +1303,7 @@ static gboolean updateBuildOptions(ProjectDataT *self) { static void sendSFTP(SFTPT *sftp) { ProjectDataT *self = (ProjectDataT *)sftp->userData; SFTPT *last; + SFTPT *next; char *target; char *name; @@ -1304,7 +1329,11 @@ static void sendSFTP(SFTPT *sftp) { } else { // Finished! message(MSG_INFO, "Waiting for build to complete"); - g_timeout_add_seconds(5, waitForBuild, last->sshData); + // "last" will be deleted by sshSFTPUpdate so copy what we need to "next". + next = NEW(SFTPT); + next->sshData = last->sshData; + next->userData = last->userData; + g_timeout_add_seconds(5, waitForBuild, next); } } @@ -1380,26 +1409,37 @@ static void targetArrayDelete(TargetT ***array) { static gboolean waitForBuild(gpointer userData) { - SSHT *ssh = (SSHT *)userData; - SFTPT *sftp; - char *temp; + SFTPT *last = (SFTPT *)userData; + ProjectDataT *self = (ProjectDataT *)last->userData; + SFTPT *sftp; + char *temp; - if (sshSFTPFileExists(ssh, "build/build.tar.bz2")) { + if (sshSFTPFileExists(last->sshData, REMOTE_BUILD_RESULTS)) { // Build complete! Next step! + //***TODO*** Delete any local build archive. message(MSG_INFO, "Retrieving build results"); - temp = utilCreateString("%s%cbuild.tar.bz2", g_get_user_config_dir(), UTIL_PATH_CHAR); - sftp = sshSFTPReceive(ssh, "build/build.tar.bz2", temp, receiveSFTP); + temp = utilCreateString("%s%s", self->windowData.path, LOCAL_BUILD_RESULTS); + sftp = sshSFTPReceive(last->sshData, REMOTE_BUILD_RESULTS, temp, receiveSFTP); DEL(temp); if (!sftp) { message(MSG_ERROR, "Receiving build from build server - FAILED"); message(MSG_INFO, "Build canceled"); - sshDisconnect(&ssh); + sshDisconnect(&last->sshData); + } else { + sftp->userData = last->userData; } + + // Finally finished with "last" + DEL(last->localName); + DEL(last->remoteName); + DEL(last); + return G_SOURCE_REMOVE; } // Keep waiting. //***TODO*** Add a timeout of some kind here and sshDisconnect. + return G_SOURCE_CONTINUE; } diff --git a/src/ssh.c b/src/ssh.c index f932808..1c2aa1d 100644 --- a/src/ssh.c +++ b/src/ssh.c @@ -133,6 +133,35 @@ void sshDisconnect(SSHT **sshData) { } +gboolean sshSFTPDelete(SSHT *sshData, char *target) { + LIBSSH2_SFTP *session; + int rc; + gboolean result = TRUE; + + do { + session = libssh2_sftp_init(sshData->session); + if (!session) { + if (libssh2_session_last_errno(sshData->session) != LIBSSH2_ERROR_EAGAIN) { + return FALSE; + } + } + } while (!session); + + do { + rc = libssh2_sftp_unlink(session, target); + if (rc == 0) break; + if (rc != LIBSSH2_ERROR_EAGAIN) { + result = FALSE; + break; + } + } while (TRUE); + + libssh2_sftp_shutdown(session); + + return result; +} + + /* int sshExecute(SSHT *sshData, char *command, char **output) { int rc; diff --git a/src/utils.c b/src/utils.c index d5e2e63..4d87e4d 100644 --- a/src/utils.c +++ b/src/utils.c @@ -41,11 +41,15 @@ typedef struct WindowListS { } WindowListT; -static WindowListT *_windowList = NULL; +static ArchiveT **_activeArchives = NULL; +static WindowListT *_windowList = NULL; char __utilFilenameBuffer[FILENAME_MAX]; +gboolean utilDecompressUpdate(gpointer userData); // Not static + + char *utilCreateString(char *format, ...) { va_list args; char *string; @@ -76,6 +80,176 @@ char *utilCreateStringVArgs(char *format, va_list args) { } +ArchiveT *utilDecompress(char *archive, char *outPath, archiveCallback callback, void *userData) { + ArchiveT *a = NULL; + int i; + + // Do not allow two of the same archive to be started. + for (i=0; iarchive, archive) == 0) { + // Return existing transfer handle. + return _activeArchives[i]; + } + } + + a = NEW(ArchiveT); + + a->in = archive_read_new(); + archive_read_support_filter_bzip2(a->in); + archive_read_support_format_gnutar(a->in); + + a->out = archive_write_disk_new(); + archive_write_disk_set_options(a->out, 0); + archive_write_disk_set_standard_lookup(a->out); + + a->entry = NULL; + a->result = 0; + a->compression = 0; + a->fileCount = 0; + a->compressed = 0; + a->uncompressed = 0; + a->archive = utilFileBasename(archive); + a->outPath = strdup(outPath); + a->currentFile = NULL; + a->copying = FALSE; + a->finished = FALSE; + a->success = FALSE; + a->callback = callback; + a->userData = userData; + a->size = utilFileSize(archive); + + a->result = archive_read_open_filename(a->in, archive, 16384); + if (a->result == ARCHIVE_OK) { + arrput(_activeArchives, a); + return a; + } + + debug("%s\n", archive_error_string(a->in)); + + DEL(a->archive); + DEL(a->outPath); + DEL(a); + + return NULL; +} + + +gboolean utilDecompressUpdate(gpointer userData) { + ArchiveT *a; + const void *buff; + size_t size; + la_int64_t offset; + int i; + int toClose = -1; + char *fullOutputPath; + + (void)userData; + + // Anything to process? + for (i=0; ifinished) { + a->success = FALSE; + toClose = i; + break; + } + // Update stats. + if (a->in != NULL) { + a->compressed = archive_filter_bytes(a->in, -1); + a->uncompressed = archive_filter_bytes(a->in, 0); + if (a->compressed > a->uncompressed) { + a->compression = 0; + } else { + a->compression = (int)((a->uncompressed - a->compressed) * 100 / a->uncompressed); + } + a->fileCount = archive_file_count(a->in); + } + + if (a->entry != NULL) { + DEL(a->currentFile); + a->currentFile = strdup(archive_entry_pathname(a->entry)); + } + // Are we unpacking, or do we need the next file? + if (a->copying) { + a->result = archive_read_data_block(a->in, &buff, &size, &offset); + if (a->result == ARCHIVE_EOF) { + a->copying = FALSE; + a->result = archive_write_finish_entry(a->out); + if (a->result < ARCHIVE_WARN) { + a->finished = TRUE; + a->success = FALSE; + toClose = i; + } + break; + } + if (a->result < ARCHIVE_OK) { + a->finished = TRUE; + a->success = FALSE; + toClose = i; + break; + } + a->result = archive_write_data_block(a->out, buff, size, offset); + if (a->result < ARCHIVE_OK) { + a->finished = TRUE; + a->success = FALSE; + toClose = i; + break; + } + } else { + // Get next file to unpack. + a->result = archive_read_next_header(a->in, &a->entry); + if (a->result == ARCHIVE_EOF) { + a->finished = TRUE; + a->success = TRUE; + toClose = i; + break; + } + if (a->result < ARCHIVE_WARN) { + a->finished = TRUE; + a->success = FALSE; + toClose = i; + break; + } + // Add path to output filename. + fullOutputPath = utilCreateString("%s%s", a->outPath, archive_entry_pathname(a->entry)); + archive_entry_set_pathname(a->entry, fullOutputPath); + DEL(fullOutputPath); + // Create output file. + a->result = archive_write_header(a->out, a->entry); + if (a->result != ARCHIVE_OK) { + a->finished = TRUE; + a->success = FALSE; + toClose = i; + break; + } + // Is there data to decompress? + if (archive_entry_size(a->entry) > 0) { + // Start copying decompressed data. + a->copying = TRUE; + } + } + } + + if (toClose >= 0) { + a = _activeArchives[toClose]; + if (a->callback) { + a->callback(a); + } + archive_read_close(a->in); + archive_read_free(a->in); + archive_write_close(a->out); + archive_write_free(a->out); + DEL(a->archive); + DEL(a->outPath); + DEL(a->currentFile); + DEL(a); + } + + return G_SOURCE_CONTINUE; +} + + char *utilDeobfuscateASCII(char *obfuscated) { char *deobfuscated = NULL; char *hostname; @@ -275,8 +449,8 @@ gboolean utilFileOpen(WindowDataT *self, char *extension, char *what) { char *utilFilePath(char *filename) { - int i; - char *temp = NULL; + size_t i; + char *temp = NULL; if (filename) { // Derive the path from the filename. @@ -640,6 +814,16 @@ void utilSetDirty(WindowDataT *self, gboolean dirty) { } +void utilShutdown(void) { + g_idle_remove_by_data(utilDecompressUpdate); +} + + +void utilStartup(void) { + g_idle_add(utilDecompressUpdate, utilDecompressUpdate); +} + + void utilUpdatePath(WindowDataT *self) { DEL(self->path); self->path = utilFilePath(self->filename); diff --git a/thirdparty/bzip2-bzip2-1.0.8.tar.bz2 b/thirdparty/bzip2-bzip2-1.0.8.tar.bz2 new file mode 100644 index 0000000..c5458cd --- /dev/null +++ b/thirdparty/bzip2-bzip2-1.0.8.tar.bz2 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a1b75f370f6963fc7bc2f78b31c754036c822ac77d7cc618e79bb834a49f482 +size 337936 diff --git a/thirdparty/libarchive-3.6.2.tar.gz b/thirdparty/libarchive-3.6.2.tar.gz new file mode 100644 index 0000000..b9fba87 --- /dev/null +++ b/thirdparty/libarchive-3.6.2.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba6d02f15ba04aba9c23fd5f236bb234eab9d5209e95d1c4df85c44d5f19b9b3 +size 7428511 diff --git a/tools/prebuild.sh b/tools/prebuild.sh index 079dd61..bd84146 100755 --- a/tools/prebuild.sh +++ b/tools/prebuild.sh @@ -176,6 +176,28 @@ pushd "${ROOT}" || exit &> /dev/null popd || true &> /dev/null fi + if [[ ! -f ${INSTALLED}/lib/libbz2.a ]]; then + echo Building Dependency: libbz2... + tar xjf ${THIRDPARTY}/bzip2-bzip2-1.0.8.tar.bz2 + pushd bzip2-bzip2-1.0.8 || exit &> /dev/null + make libbz2.a PREFIX=${INSTALLED} + make install libbz2.a PREFIX=${INSTALLED} + popd || true &> /dev/null + fi + + if [[ ! -f ${INSTALLED}/lib/libarchive.a ]]; then + echo Building Dependency: libarchive... + tar xzf ${THIRDPARTY}/libarchive-3.6.2.tar.gz + pushd libarchive-3.6.2 || exit &> /dev/null + ./configure \ + --without-xml2 \ + --prefix=${INSTALLED} \ + --enable-static + make + make install + popd || true &> /dev/null + fi + popd || true &> /dev/null echo Generating UI Embedded Code...