From 8de2ffc18fcf4c544e0ceaf812c62c42fd218c98 Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Wed, 3 May 2023 20:44:44 -0500 Subject: [PATCH] Builds being sent and received over SFTP! Now to unpack them! --- include/ssh.h | 11 ++-- include/utils.h | 1 + src/messages.c | 46 +++++++++++--- src/project.c | 106 +++++++++++++++++++++++++++----- src/ssh.c | 158 ++++++++++++++++++++++++++++++++---------------- src/utils.c | 13 ++++ 6 files changed, 253 insertions(+), 82 deletions(-) diff --git a/include/ssh.h b/include/ssh.h index 9127966..0d760b5 100644 --- a/include/ssh.h +++ b/include/ssh.h @@ -44,9 +44,7 @@ typedef struct SSHS { int sock; int result; gboolean finished; - char *title; SSHCallback callback; - GtkWidget *status; } SSHT; @@ -61,14 +59,12 @@ typedef struct SFTPS { uint64_t fileSize; uint64_t fileSent; size_t nread; - char *source; - char *target; - char *title; + char *localName; + char *remoteName; gboolean uploading; gboolean readMore; gboolean finished; gboolean success; - GtkWidget *status; SFTPCallback callback; } SFTPT; @@ -77,7 +73,8 @@ SSHT *sshConnect(char *hostname, uint16_t port, char *user, char *password); void sshDisconnect(SSHT **sshData); //int sshExecute(SSHT *sshData, char *command, char **output); gboolean sshSFTPFileExists(SSHT *sshData, char *target); -SFTPT *sshSFTPSend(char *title, SSHT *sshData, char *source, char *target, SFTPCallback callback); +SFTPT *sshSFTPReceive(SSHT *sshData, char *remote, char *local, SFTPCallback callback); +SFTPT *sshSFTPSend(SSHT *sshData, char *local, char *remote, SFTPCallback callback); void sshShutdown(void); void sshStartup(void); diff --git a/include/utils.h b/include/utils.h index 8b86228..6806a64 100644 --- a/include/utils.h +++ b/include/utils.h @@ -53,6 +53,7 @@ gboolean utilFileSaveAs(WindowDataT *self, char *extension, char *what); gboolean utilFileSaveOtherAs(WindowDataT *self, char *extension, char *what, char **filename); size_t utilFileSize(char *filename); GtkWidget *utilFindChildWidget(GtkWidget *parent, const gchar *name); +void utilForceUpdate(void); char *utilGetToken(char *input, char *delimit, char *openblock, char *closeblock); WindowDataT *utilGetWindowData(GtkWidget *window); gboolean utilGetWidgetsFromMemory(char *resource, char *name[], GtkWidget **widgets[], gpointer userData); diff --git a/src/messages.c b/src/messages.c index aa0d1dc..66c7d94 100644 --- a/src/messages.c +++ b/src/messages.c @@ -26,8 +26,10 @@ static GtkWidget *_lstMessages = NULL; +static int _autoScroll = 0; +static gboolean messagesScroll(gpointer userData); EVENT gboolean winMessagesClose(GtkWidget *object, gpointer userData); static void winMessagesDelete(gpointer userData); @@ -43,19 +45,19 @@ void message(MessageTypesT level, char *format, ...) { NULL, &_lstMessages }; - GtkWidget *row; - GtkWidget *box; - GtkWidget *label; - va_list args; - char *string; - char *msg; - char *tok; - char *labels[MSG_COUNT] = { + GtkWidget *row; + GtkWidget *box; + GtkWidget *label; + va_list args; + char *string; + char *msg; + char *tok; + char *labels[MSG_COUNT] = { " Info:", "Warning:", " Error:", " Severe:" - }; + }; // Do we need to open this window? if (!_lstMessages) { @@ -71,6 +73,9 @@ void message(MessageTypesT level, char *format, ...) { // Show window. gtk_widget_show_all(self->window); + + // Set up automatic scrolling. + g_idle_add(messagesScroll, messagesScroll); } // Display message. @@ -96,13 +101,35 @@ void message(MessageTypesT level, char *format, ...) { gtk_container_add(GTK_CONTAINER(row), box); gtk_list_box_insert(GTK_LIST_BOX(_lstMessages), row, -1); + // Force paint. gtk_widget_show_all(row); + utilForceUpdate(); + _autoScroll = 5; // Try 5 times to show the new row. tok = strtok(NULL, "\n"); } } +static gboolean messagesScroll(gpointer userData) { + GtkAdjustment *adjustment; + + (void)userData; + + if (_autoScroll <= 0) return G_SOURCE_CONTINUE; + + // Scroll to show new line. + adjustment = gtk_list_box_get_adjustment(GTK_LIST_BOX(_lstMessages)); + gtk_adjustment_set_value(adjustment, gtk_adjustment_get_upper(adjustment)); + gtk_widget_show_all(_lstMessages); + utilForceUpdate(); + + _autoScroll--; + + return G_SOURCE_CONTINUE; +} + + EVENT gboolean winMessagesClose(GtkWidget *object, gpointer userData) { // userData is not reliable due to util indirectly calling us. WindowDataT *self = (WindowDataT *)utilGetWindowData(object); @@ -115,6 +142,7 @@ EVENT gboolean winMessagesClose(GtkWidget *object, gpointer userData) { static void winMessagesDelete(gpointer userData) { + g_idle_remove_by_data(messagesScroll); utilWindowUnRegister(userData); _lstMessages = NULL; DEL(userData); diff --git a/src/project.c b/src/project.c index 5f2399d..04cb9c9 100644 --- a/src/project.c +++ b/src/project.c @@ -132,12 +132,14 @@ EVENT void menuProjectBuildTargets(GtkWidget *object, gpointer userData EVENT void menuProjectBuildCookRecipes(GtkWidget *object, gpointer userData); EVENT void menuProjectBuildBuild(GtkWidget *object, gpointer userData); EVENT void menuProjectHelpProject(GtkWidget *object, gpointer userData); +static void receiveSFTP(SFTPT *sftp); static void saveConfig(ProjectDataT *self); static void sendSFTP(SFTPT *sftp); static TargetT **targetArrayCopy(TargetT **targets); static void targetArrayDelete(TargetT ***array); EVENT void treeProjectRowActivated(GtkTreeView *treeView, GtkTreePath *path, GtkTreeViewColumn *column, gpointer userData); static gboolean updateBuildOptions(ProjectDataT *self); +static gboolean waitForBuild(gpointer userData); EVENT gboolean winProjectClose(GtkWidget *object, gpointer userData); static void winProjectDelete(gpointer userData); @@ -955,20 +957,28 @@ EVENT void menuProjectBuildBuild(GtkWidget *object, gpointer userData) { gboolean archPrinted; FILE *out; + menuProjectBuildCookRecipes(object, userData); + ssh = NEW(SSHT); ssh = sshConnect(self->buildHost, self->buildSSHPort, self->buildUser, self->buildPassword); if (!ssh) { - message(MSG_ERROR, "Unable to connect to SSH port."); + message(MSG_ERROR, "Unable to connect to SSH port"); return; } + // Do we have a build package to clean up? + if (sshSFTPFileExists(ssh, "build/build.tar.bz2")) { + message(MSG_INFO, "Removing stale build results"); + } + // Generate build.start. buildStart = utilCreateString("%sbuild.start", self->windowData.path); out = fopen(buildStart, "wt"); if (!out) { - message(MSG_SEVERE, "Unable to write temporary build.start file."); + message(MSG_SEVERE, "Unable to write temporary build.start file"); unlink(buildStart); // Just in case. + message(MSG_INFO, "Build canceled"); sshDisconnect(&ssh); return; } @@ -1085,6 +1095,25 @@ gboolean projectAddToTree(ProjectDataT *self, char *filename) { } +static void receiveSFTP(SFTPT *sftp) { + ProjectDataT *self = (ProjectDataT *)sftp->userData; + + if (sftp->finished) { + if (sftp->success) { + //***TODO*** Unpack results + message(MSG_INFO, "Unpacking build"); + } else { + message(MSG_ERROR, "Receiving %s from build server - FAILED", sftp->remoteName); + message(MSG_INFO, "Build canceled"); + sshDisconnect(&sftp->sshData); + } + DEL(sftp->localName); + DEL(sftp->remoteName); + DEL(sftp); + } +} + + static void saveConfig(ProjectDataT *self) { char *temp; FILE *out; @@ -1249,32 +1278,56 @@ static gboolean updateBuildOptions(ProjectDataT *self) { static void sendSFTP(SFTPT *sftp) { ProjectDataT *self = (ProjectDataT *)sftp->userData; - SFTPT *next; - char *title; + SFTPT *last; char *target; + char *name; + + last = sftp; // If source and target are missing, this is the first file - start sending. // Or, did the current transfer succeed? If so, send next file. - if ((sftp->source == NULL && sftp->target == NULL) || (sftp->finished && sftp->success)) { + if ((sftp->localName == NULL && sftp->remoteName == NULL) || (sftp->finished && sftp->success)) { + if (sftp->localName == NULL && sftp->remoteName == NULL) { + message(MSG_INFO, "Building %s", self->windowData.filename); + } // Are there more files to send? if (arrlen(_sendList) > 0) { // Send next entry in the file list. - title = utilCreateString("Building %s", self->windowData.filename); - target = utilFileBasename(_sendList[0]); - sftp = sshSFTPSend(title, sftp->sshData, _sendList[0], target, sendSFTP); + name = utilFileBasename(_sendList[0]); + target = utilCreateString("build/%s", name); + message(MSG_INFO, "Sending %s to build server", name); + sftp = sshSFTPSend(last->sshData, _sendList[0], target, sendSFTP); + sftp->userData = self; DEL(target); - DEL(title); + DEL(name); arrdel(_sendList, 0); - return; } else { // Finished! - //***TODO*** + message(MSG_INFO, "Waiting for build to complete"); + g_timeout_add_seconds(5, waitForBuild, last->sshData); } } - // Did the transfer fail? - if (sftp->finished && !sftp->success) { - //***TODO*** + // Do we have a transfer? Did the transfer fail? + if ((!sftp) || (sftp->finished && !sftp->success)) { + if (sftp) { + message(MSG_ERROR, "Sending %s to build server - FAILED", sftp->remoteName); + } + message(MSG_INFO, "Build canceled"); + sshDisconnect(&last->sshData); + // Clear any remaining transfers. + while (arrlen(_sendList) > 0) { + arrdel(_sendList, 0); + } + ARRFREE(_sendList); + } + + // Do we need to free this? + if (last->finished) { + // Do not delete last->sshData as we continue to need it until everything is finished. + DEL(last->localName); + DEL(last->remoteName); + DEL(last); } } @@ -1326,6 +1379,31 @@ static void targetArrayDelete(TargetT ***array) { } +static gboolean waitForBuild(gpointer userData) { + SSHT *ssh = (SSHT *)userData; + SFTPT *sftp; + char *temp; + + if (sshSFTPFileExists(ssh, "build/build.tar.bz2")) { + // Build complete! Next step! + message(MSG_INFO, "Retrieving build results"); + temp = utilCreateString("%s%cbuild.tar.bz2", g_get_user_config_dir(), UTIL_PATH_CHAR); + sftp = sshSFTPReceive(ssh, "build/build.tar.bz2", temp, receiveSFTP); + DEL(temp); + if (!sftp) { + message(MSG_ERROR, "Receiving build from build server - FAILED"); + message(MSG_INFO, "Build canceled"); + sshDisconnect(&ssh); + } + return G_SOURCE_REMOVE; + } + + // Keep waiting. + //***TODO*** Add a timeout of some kind here and sshDisconnect. + return G_SOURCE_CONTINUE; +} + + EVENT gboolean winProjectClose(GtkWidget *object, gpointer userData) { // userData is not reliable due to menuVectorFileClose and util indirectly calling us. ProjectDataT *self = (ProjectDataT *)utilGetWindowData(object); diff --git a/src/ssh.c b/src/ssh.c index 535f6ac..f932808 100644 --- a/src/ssh.c +++ b/src/ssh.c @@ -47,9 +47,10 @@ static char *_buffer = NULL; static int _bufferLen = 0; -gboolean sshUpdate(gpointer userData); // Not static -gboolean sshSFTPUpdate(gpointer userData); // Not static -static int sshWaitSocket(int socket_fd, LIBSSH2_SESSION *session); +gboolean sshUpdate(gpointer userData); // Not static +static SFTPT *sshSFTPTransfer(SSHT *sshData, char *local, char *remote, SFTPCallback callback, gboolean uploading); +gboolean sshSFTPUpdate(gpointer userData); // Not static +static int sshWaitSocket(int socket_fd, LIBSSH2_SESSION *session); SSHT *sshConnect(char *hostname, uint16_t port, char *user, char *password) { @@ -220,6 +221,7 @@ gboolean sshSFTPFileExists(SSHT *sshData, char *target) { LIBSSH2_SFTP_ATTRIBUTES attrs; char *path; char *file; + char *temp; char buffer[512]; int rc; gboolean result = FALSE; @@ -233,8 +235,25 @@ gboolean sshSFTPFileExists(SSHT *sshData, char *target) { } } while (!session); - path = utilFilePath(target); - file = utilFileBasename(target); + // Split filename and path. + // Cannot use utilFilePath and friends because this is a remote path. + temp = strdup(target); // Make copy in case they pass in a constant. + path = strdup("/"); // Set some defaults. + file = strdup(target); // Set some defaults. + rc = (int)strlen(temp); + while (rc > 0) { + if (temp[rc] == '/') { + // Found it. + temp[rc] = 0; + DEL(path); + DEL(file); + path = strdup(temp); + file = strdup(&temp[rc + 1]); + break; + } + rc--; + } + DEL(temp); do { handle = libssh2_sftp_opendir(session, path); @@ -276,13 +295,23 @@ gboolean sshSFTPFileExists(SSHT *sshData, char *target) { } -SFTPT *sshSFTPSend(char *title, SSHT *sshData, char *source, char *target, SFTPCallback callback) { +SFTPT *sshSFTPReceive(SSHT *sshData, char *remote, char *local, SFTPCallback callback) { + return sshSFTPTransfer(sshData, local, remote, callback, FALSE); +} + + +SFTPT *sshSFTPSend(SSHT *sshData, char *local, char *remote, SFTPCallback callback) { + return sshSFTPTransfer(sshData, local, remote, callback, TRUE); +} + + +static SFTPT *sshSFTPTransfer(SSHT *sshData, char *local, char *remote, SFTPCallback callback, gboolean uploading) { SFTPT *sftp = NULL; int i; // Do not allow two of the same transfers to be started. for (i=0; isource, source) == 0) && (strcmp(_activeSFTPs[i]->target, target) == 0)) { + if ((strcmp(_activeSFTPs[i]->localName, local) == 0) && (strcmp(_activeSFTPs[i]->remoteName, remote) == 0)) { // Return existing transfer handle. return _activeSFTPs[i]; } @@ -291,21 +320,20 @@ SFTPT *sshSFTPSend(char *title, SSHT *sshData, char *source, char *target, SFTPC sftp = NEW(SFTPT); if (sftp) { - sftp->fileSize = utilFileSize(source); - sftp->local = fopen(source, "rb"); + sftp->fileSize = uploading ? utilFileSize(local) : 0; + sftp->local = fopen(local, uploading ? "rb" : "wb"); if (sftp->local) { - sftp->fileSent = 0; - sftp->nread = 0; - sftp->ptr = NULL; - sftp->title = strdup(title); - sftp->source = strdup(source); - sftp->target = strdup(target); - sftp->sshData = sshData; - sftp->uploading = TRUE; - sftp->readMore = TRUE; - sftp->finished = FALSE; - sftp->success = FALSE; - sftp->callback = callback; + sftp->fileSent = 0; + sftp->nread = 0; + sftp->ptr = NULL; + sftp->localName = strdup(local); + sftp->remoteName = strdup(remote); + sftp->sshData = sshData; + sftp->uploading = uploading; + sftp->readMore = TRUE; + sftp->finished = FALSE; + sftp->success = FALSE; + sftp->callback = callback; do { sftp->session = libssh2_sftp_init(sshData->session); if (!sftp->session) { @@ -315,7 +343,17 @@ SFTPT *sshSFTPSend(char *title, SSHT *sshData, char *source, char *target, SFTPC } } while (!sftp->session); do { - sftp->handle = libssh2_sftp_open(sftp->session, sftp->target, LIBSSH2_FXF_WRITE | LIBSSH2_FXF_CREAT | LIBSSH2_FXF_TRUNC, LIBSSH2_SFTP_S_IRUSR | LIBSSH2_SFTP_S_IWUSR | LIBSSH2_SFTP_S_IRGRP | LIBSSH2_SFTP_S_IROTH); + if (uploading) { + sftp->handle = libssh2_sftp_open(sftp->session, sftp->remoteName, + LIBSSH2_FXF_WRITE | LIBSSH2_FXF_CREAT | LIBSSH2_FXF_TRUNC, + LIBSSH2_SFTP_S_IRUSR | LIBSSH2_SFTP_S_IWUSR | LIBSSH2_SFTP_S_IRGRP | + LIBSSH2_SFTP_S_IROTH); + } else { + sftp->handle = libssh2_sftp_open(sftp->session, sftp->remoteName, + LIBSSH2_FXF_READ, + LIBSSH2_SFTP_S_IRUSR | LIBSSH2_SFTP_S_IWUSR | LIBSSH2_SFTP_S_IRGRP | + LIBSSH2_SFTP_S_IROTH); + } if (!sftp->handle) { if (libssh2_session_last_errno(sshData->session) != LIBSSH2_ERROR_EAGAIN) { break; @@ -323,8 +361,6 @@ SFTPT *sshSFTPSend(char *title, SSHT *sshData, char *source, char *target, SFTPC } } while (!sftp->handle); if (sftp->handle) { - //***TODO*** SFTP Status Dialog - //sftp->status = new SFTPStatus(sftp); arrput(_activeSFTPs, sftp); return sftp; } else { @@ -344,60 +380,78 @@ SFTPT *sshSFTPSend(char *title, SSHT *sshData, char *source, char *target, SFTPC gboolean sshSFTPUpdate(gpointer userData) { int i; int rc; - int count = 0; SFTPT *result = NULL; (void)userData; for (i=0; ireadMore) { - _activeSFTPs[i]->nread = fread(_activeSFTPs[i]->buffer, 1, sizeof(_activeSFTPs[i]->buffer), _activeSFTPs[i]->local); - if (_activeSFTPs[i]->nread <= 0) { - result = _activeSFTPs[i]; - arrdel(_activeSFTPs, i); - break; - } - _activeSFTPs[i]->ptr = _activeSFTPs[i]->buffer; - _activeSFTPs[i]->readMore = FALSE; - } + // Uploading or downloading? + if (_activeSFTPs[i]->uploading) { + // Uploading - // Is there data to send? - if (_activeSFTPs[i]->nread > 0) { - // Send data. - while ((rc = (int)libssh2_sftp_write(_activeSFTPs[i]->handle, _activeSFTPs[i]->ptr, _activeSFTPs[i]->nread)) == LIBSSH2_ERROR_EAGAIN) { - sshWaitSocket(_activeSFTPs[i]->sshData->sock, _activeSFTPs[i]->sshData->session); + // More data to read? + if (_activeSFTPs[i]->readMore) { + _activeSFTPs[i]->nread = fread(_activeSFTPs[i]->buffer, 1, sizeof(_activeSFTPs[i]->buffer), _activeSFTPs[i]->local); + if (_activeSFTPs[i]->nread <= 0) { + result = _activeSFTPs[i]; + arrdel(_activeSFTPs, i); + break; + } + _activeSFTPs[i]->ptr = _activeSFTPs[i]->buffer; + _activeSFTPs[i]->readMore = FALSE; } - if (rc >= 0) { - _activeSFTPs[i]->ptr += rc; - _activeSFTPs[i]->nread -= rc; - _activeSFTPs[i]->fileSent += rc; + + // Is there data to send? + if (_activeSFTPs[i]->nread > 0) { + // Send data. + while ((rc = (int)libssh2_sftp_write(_activeSFTPs[i]->handle, _activeSFTPs[i]->ptr, _activeSFTPs[i]->nread)) == LIBSSH2_ERROR_EAGAIN) { + sshWaitSocket(_activeSFTPs[i]->sshData->sock, _activeSFTPs[i]->sshData->session); + } + if (rc >= 0) { + _activeSFTPs[i]->ptr += rc; + _activeSFTPs[i]->nread -= rc; + _activeSFTPs[i]->fileSent += rc; + } else { + // We need more data! + _activeSFTPs[i]->readMore = TRUE; + } } else { // We need more data! _activeSFTPs[i]->readMore = TRUE; } + } else { - // We need more data! - _activeSFTPs[i]->readMore = TRUE; + // Downloading + + // Data to read? + while ((rc = libssh2_sftp_read(_activeSFTPs[i]->handle, _activeSFTPs[i]->buffer, sizeof(_activeSFTPs[i]->buffer))) == LIBSSH2_ERROR_EAGAIN) { + sshWaitSocket(_activeSFTPs[i]->sshData->sock, _activeSFTPs[i]->sshData->session); + } + if (rc > 0) { + _activeSFTPs[i]->fileSize += rc; + fwrite(_activeSFTPs[i]->buffer, 1, rc, _activeSFTPs[i]->local); + } else { + result = _activeSFTPs[i]; + arrdel(_activeSFTPs, i); + break; + } + } } if (result) { result->finished = TRUE; result->success = TRUE; + fclose(result->local); libssh2_sftp_close(result->handle); libssh2_sftp_shutdown(result->session); - //***TODO*** Display in status dialog - //result->status->Finished(); - //result->status->Close(); if (result->callback) { result->callback(result); } DEL(result); } - return count; + return G_SOURCE_CONTINUE; } diff --git a/src/utils.c b/src/utils.c index f329b23..d5e2e63 100644 --- a/src/utils.c +++ b/src/utils.c @@ -409,6 +409,16 @@ GtkWidget *utilFindChildWidget(GtkWidget *parent, const gchar *name) { } +void utilForceUpdate(void) { + //***TODO*** This is supposed to force any pending paint operations. + // It doesn't work. + + //while (g_main_context_pending(NULL)) { + g_main_context_iteration(NULL,FALSE); + //} +} + + char *utilGetToken(char *input, char *delimit, char *openblock, char *closeblock) { static char *token = NULL; char *lead = NULL; @@ -642,6 +652,9 @@ void utilWindowRegister(gpointer windowData) { // Grab title. w->title = strdup(gtk_window_get_title(GTK_WINDOW(w->window))); + gtk_widget_show_all(w->window); + utilForceUpdate(); + hmput(_windowList, w->window, windowData); debug("Window Registered: %ld\n", hmlen(_windowList)); }