From d15b264ef6f8bebf451620e99e06a85fb4308be3 Mon Sep 17 00:00:00 2001 From: Scott Duensing Date: Mon, 14 Feb 2022 20:13:42 -0600 Subject: [PATCH] File transfers working. --- client/src/file.c | 127 ++++++++++++++++++++++++++++---------- client/src/hangup.c | 2 +- client/src/main.c | 8 +-- client/src/menu.c | 2 +- client/src/system/cache.c | 6 +- kpmpgsmkii.pro | 4 +- schema.sql | 12 ++-- server/src/client.c | 4 ++ server/src/client.h | 6 +- server/src/client/file.c | 65 +++++++++++-------- server/src/database.c | 3 + server/src/server.c | 2 + shared/packets.h | 5 +- 13 files changed, 167 insertions(+), 79 deletions(-) diff --git a/client/src/file.c b/client/src/file.c index c7edd04..5841d07 100644 --- a/client/src/file.c +++ b/client/src/file.c @@ -20,6 +20,7 @@ #include "network.h" #include "packet.h" +#include "array.h" #include "taglist.h" #include "window.h" @@ -28,12 +29,22 @@ #include "file.h" +// All this queue nonsense allows you to request more +// downloads while downloads are currently running. +typedef struct FileListS { + char **files; + fileCallback callback; +} FileListT; + + static uint8_t _channel = 0; static uint8_t _dialogVisible = 0; -static int16_t _index = -1; -static char **_fileList = NULL; +static FileListT **_fileList = NULL; +static FileListT *_current = NULL; +static uint32_t _currentLength = 0; +static char *_currentSha256 = NULL; +static char *_file = NULL; static FILE *_handle = NULL; -static fileCallback _done = NULL; static WindowT *_winFile = NULL; static LabelT *_lblFile = NULL; @@ -45,11 +56,24 @@ static void packetHandler(PacketDecodeDataT *packet); void fileCacheCheck(fileCallback callback, char *vpaths[]) { - _done = callback; - _fileList = vpaths; - _index = -1; - _dialogVisible = 0; - _channel = netChannelGet(packetHandler); + FileListT *newList = NULL; + uint16_t i = 0; + + // Add new entries to anything already in the queue. + NEW(FileListT, newList); + newList->callback = callback; + newList->files = NULL; + while (vpaths[i] != NULL) { + arrput(newList->files, strdup(vpaths[i])); + i++; + } + arrput(_fileList, newList); + + // New queue? + if (_channel == 0) { + _dialogVisible = 0; + _channel = netChannelGet(packetHandler); + } fileCheckNext(); } @@ -60,25 +84,57 @@ static void fileCheckNext(void) { char *packetData = NULL; uint16_t length = 0; uint32_t temp = 0; + uint8_t doRecheck = 0; char *shaPointer = NULL; static char *badSHA = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; - _index++; + do { + // This is ugly since both lists kind of depend on each other. + doRecheck = 0; - if (_fileList[_index] == NULL) { - // End of list! - if (_dialogVisible) guiDelete(D(_winFile)); - netChannelRelease(_channel); - _done(); - return; - } + // Do we need a new entry from the queue? + if (_current == NULL) { + // End of queue? + if (arrlen(_fileList) == 0) { + logWrite("End of queue.\n"); + arrfree(_fileList); + if (_dialogVisible) guiDelete(D(_winFile)); + netChannelRelease(_channel); + return; + } + + // Get next queue entry. + logWrite("Next queue entry.\n"); + _currentLength = 0; + _current = _fileList[0]; + arrdel(_fileList, 0); + } + + if (_file) DEL(_file); + + // End of current queue file list? + if (arrlen(_current->files) == 0) { + logWrite("End of files.\n"); + // End of list! + arrfree(_current->files); + _current->callback(); + DEL(_current); + // See if there's more. + doRecheck = 1; + } else { + // No. Get next file. + logWrite("Next file.\n"); + _file = _current->files[0]; + arrdel(_current->files, 0); + } + } while (doRecheck); // Check next file. temp = FILE_REQUEST_CHECK; - shaPointer = cacheSha256Get(_fileList[_index]); + shaPointer = cacheSha256Get(_file); if (!shaPointer) shaPointer = badSHA; - packetData = packetContentPack(&length, "iss", temp, shaPointer, _fileList[_index]); - logWrite("Checking [%d] [%s] [%s]\n", temp, shaPointer, _fileList[_index]); + packetData = packetContentPack(&length, "iss", temp, shaPointer, _file); + logWrite("Checking [%d] [%s] [%s]\n", temp, shaPointer, _file); encoded.control = PACKET_CONTROL_DAT; encoded.packetType = PACKET_TYPE_FILE_REQUEST; encoded.channel = _channel; @@ -117,18 +173,14 @@ static void fileShowError(char *message) { static void packetHandler(PacketDecodeDataT *packet) { FileResponseT response = 0; - uint32_t fileLength = 0; PacketEncodeDataT encoded = { 0 }; char *packetData = NULL; uint16_t length = 0; uint32_t temp = 0; - logWrite("Got packet %d\n", packet->packetType); - if (packet->packetType == PACKET_TYPE_FILE_RESPONSE) { // Extract the response type. We get more data later. packetContentUnpack(packet->data, "i", &response); - logWrite("Got response %d [%d bytes]\n", response, sizeof(response)); switch (response) { case FILE_RESPONSE_UNKNOWN: logWrite("Unknown file transfer response.\n"); @@ -142,36 +194,43 @@ static void packetHandler(PacketDecodeDataT *packet) { break; case FILE_RESPONSE_SEND: - // Get file size. + // Start of new file. Get SHA and length. logWrite("Got FILE_RESPONSE_SEND\n"); - packetContentUnpack(packet->data, "ii", &response, &fileLength); + packetContentUnpack(packet->data, "iis", &response, &_currentLength, &_currentSha256); + break; + + case FILE_RESPONSE_DATA: + // Get file size. + logWrite("Got FILE_RESPONSE_DATA\n"); // Receive new file data. if (!_dialogVisible) fileShowDialog(); // Are we starting a new file? if (_handle == NULL) { - logWrite("Opening [%s]\n", _fileList[_index]); - _handle = cacheFOpen(_fileList[_index], "wb"); + logWrite("Opening [%s]\n", _file); + _handle = cacheFOpen(_file, "wb"); if (!_handle) { fileShowError("Unable to write to cache."); break; } } - // Write data to file, skipping first two integers stored in packet. - logWrite("Writing %d bytes to [%s]\n", packet->length - 8, _fileList[_index]); - fwrite(&packet->data[8], packet->length - 8, 1, _handle); + // Write data to file, skipping first integer stored in packet. + logWrite("Writing %d bytes to [%s]\n", packet->length - 4, _file); + fwrite(&packet->data[4], packet->length - 4, 1, _handle); // Is this file complete? - if (ftell(_handle) >= fileLength) { - logWrite("Closing [%s]\n", _fileList[_index]); + if (ftell(_handle) >= _currentLength) { + logWrite("Closing [%s]\n", _file); // Close this file. cacheFClose(_handle); _handle = NULL; + // Update cache entry to include SHA. + cacheEntryAdd(_currentSha256, cacheEntryNameGet(_file), _file); // Next file! fileCheckNext(); } else { logWrite("Sending FILE_REQUEST_NEXT\n"); // Tell the server we got this bit of data, send the next. - temp = FILE_REQUEST_NEXT; - packetData = packetContentPack(&length, "i", temp); + temp = FILE_REQUEST_NEXT; + packetData = packetContentPack(&length, "i", temp); encoded.control = PACKET_CONTROL_DAT; encoded.packetType = PACKET_TYPE_FILE_REQUEST; encoded.channel = _channel; diff --git a/client/src/hangup.c b/client/src/hangup.c index 7438dbc..6e3b05e 100644 --- a/client/src/hangup.c +++ b/client/src/hangup.c @@ -96,7 +96,7 @@ static void timHangupProgress(WidgetT *widget) { case S_WAITING: // Shut down packet processing & COM port. - netShutdown(); + netPacketHandlerStop(); comClose(__configData.serialCom - 1); timerStop(t); // On to the next dialog. diff --git a/client/src/main.c b/client/src/main.c index 071d145..9469c2f 100644 --- a/client/src/main.c +++ b/client/src/main.c @@ -220,7 +220,7 @@ static void tableLoad(void) { line = (char *)malloc(4096); if (line) { // Load string cache. - cache = cacheFOpen("system:strings", "rt"); + cache = cacheFOpen("data:strings.dat", "rt"); if (cache) { while (fscanf(cache, "%s\n", line) != EOF) { p = strstr(line, "="); @@ -239,7 +239,7 @@ static void tableLoad(void) { cacheFClose(cache); } // Load integer cache. - cache = cacheFOpen("system:integers", "rt"); + cache = cacheFOpen("data:integers.dat", "rt"); if (cache) { while (fscanf(cache, "%s\n", line) != EOF) { p = strstr(line, "="); @@ -262,7 +262,7 @@ static void tableSave(void) { FILE *cache = NULL; // Save & free integer table. - cache = cacheFOpen("system:integers", "wt"); + cache = cacheFOpen("data:integers.dat", "wt"); if (cache) { if (__runtimeData.integers) { while (shlen(__runtimeData.integers) > 0) { @@ -276,7 +276,7 @@ static void tableSave(void) { } // Save & free string table. - cache = cacheFOpen("system:strings", "wt"); + cache = cacheFOpen("data:strings.dat", "wt"); if (cache) { if (__runtimeData.strings) { while (shlen(__runtimeData.strings) > 0) { diff --git a/client/src/menu.c b/client/src/menu.c index e972c9b..675fb3f 100644 --- a/client/src/menu.c +++ b/client/src/menu.c @@ -53,7 +53,7 @@ static void btnLogoffClick(WidgetT *widget) { (void)widget; setButtons(0); - msgBoxTwo("Cancel?", MSGBOX_ICON_QUESTION, "Cancel login?\n \nThis will disconnect you from the server.", "Okay", btnMsgBoxLogoff, "Cancel", btnMsgBoxLogoff); + msgBoxTwo("Logoff?", MSGBOX_ICON_QUESTION, "This will disconnect you from the server.", "Okay", btnMsgBoxLogoff, "Cancel", btnMsgBoxLogoff); } diff --git a/client/src/system/cache.c b/client/src/system/cache.c index 7a86f6f..2264a18 100644 --- a/client/src/system/cache.c +++ b/client/src/system/cache.c @@ -115,6 +115,8 @@ uint8_t cacheEntryAdd(char *sha256, char *entryName, char *virtualPath) { // Index format is: SHA256 ENTRYNAME VIRTUALPATH + logWrite("Adding SHA [%s] Entry [%s] VPath [%s]\n", sha256, entryName, virtualPath); + sprintf(index, "CACHE%cINDEX.DAT", OS_PATH_SLASH); sprintf(indexNew, "CACHE%cINDEX.NEW", OS_PATH_SLASH); @@ -243,7 +245,7 @@ void cacheFClose(FILE *handle) { static char *cacheFieldGet(char *virtualPath, uint8_t field) { FILE *in = NULL; char index[16] = { 0 }; - static char buffer[1024] = { 0 }; + static char buffer[2048] = { 0 }; static char *name = NULL; static char *path = NULL; static char *result = NULL; @@ -258,7 +260,7 @@ static char *cacheFieldGet(char *virtualPath, uint8_t field) { if (osFileExists(index)) { in = fopen(index, "rt"); if (in) { - while ((fgets(buffer, 1024, in) != 0) && result == NULL) { + while (result == NULL && (fgets(buffer, 2048, in) != 0)) { name = strstr(buffer, " "); *name = 0; name++; diff --git a/kpmpgsmkii.pro b/kpmpgsmkii.pro index f82183f..01169bb 100644 --- a/kpmpgsmkii.pro +++ b/kpmpgsmkii.pro @@ -21,8 +21,8 @@ TEMPLATE = subdirs CONFIG *= ORDERED SUBDIRS = \ - client \ - server + client +# server # precache # font # primes diff --git a/schema.sql b/schema.sql index eae6533..597ccca 100644 --- a/schema.sql +++ b/schema.sql @@ -44,13 +44,13 @@ DROP TABLE IF EXISTS `files`; CREATE TABLE `files` ( `id` int(11) NOT NULL AUTO_INCREMENT, `realPath` varchar(1024) NOT NULL, - `virtualPath` varchar(1024) NOT NULL, + `virtualPath` varchar(512) NOT NULL, `sha256` varchar(64) NOT NULL, `description` varchar(255) NOT NULL, PRIMARY KEY (`id`), UNIQUE KEY `virtualPath` (`virtualPath`), UNIQUE KEY `realPath` (`realPath`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; -- @@ -99,14 +99,14 @@ CREATE TABLE `users` ( `first` varchar(64) NOT NULL, `last` varchar(64) NOT NULL, `user` varchar(64) NOT NULL, - `pass` varchar(64) NOT NULL, + `pass` varchar(128) NOT NULL, `email` varchar(256) NOT NULL, `enabled` tinyint(1) NOT NULL DEFAULT 0, - `notes` text NOT NULL, + `notes` text NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `emailIDX` (`email`), UNIQUE KEY `userIDX` (`user`) USING BTREE -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; /*!40101 SET character_set_client = @saved_cs_client */; /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; @@ -118,4 +118,4 @@ CREATE TABLE `users` ( /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; --- Dump completed on 2022-02-09 15:58:23 +-- Dump completed on 2022-02-14 20:13:22 diff --git a/server/src/client.c b/server/src/client.c index 2cae1eb..b0b9fa4 100644 --- a/server/src/client.c +++ b/server/src/client.c @@ -179,8 +179,12 @@ void *clientThread(void *data) { arrdel(client->packetQueue, 0); } arrfree(client->packetQueue); + pthread_mutex_destroy(&client->packetQueueMutex); packetThreadDataDestroy(&client->packetThreadData); + + if (client->handle) fclose(client->handle); + DEL(client); pthread_exit(NULL); diff --git a/server/src/client.h b/server/src/client.h index 6849667..6189bb7 100644 --- a/server/src/client.h +++ b/server/src/client.h @@ -51,9 +51,9 @@ typedef struct ClientThreadS { double pingLow; void *peer; // User State Stuff. - uint8_t authenticated; - FILE *handle; // ***TODO*** Needs to support more than one file transfer at a time. - uint32_t fileSize; // Length of current file being transferred. + uint8_t authenticated; // Is the user logged in? + FILE *handle; // Handle of file currently being transferred to client. + uint32_t fileSize; // Length of current file being transferred. } ClientThreadT; diff --git a/server/src/client/file.c b/server/src/client/file.c index 1ded8f9..4b2e418 100644 --- a/server/src/client/file.c +++ b/server/src/client/file.c @@ -75,23 +75,30 @@ static void clientApiFileRequestCheck(ClientThreadT *client, PacketDecodeDataT * PacketEncodeDataT encoded = { 0 }; char *packetData = NULL; uint16_t length = 0; + char shaInDB[128] = { 0 }; char buffer[1024] = { 0 }; - char buffer2[1024] = { 0 }; + char buffer2[2048] = { 0 }; uint32_t temp = 0; logWrite("Got FILE_REQUEST_CHECK\n\r"); + // Is something still open? + if (client->handle) { + fclose(client->handle); + client->handle = NULL; + } + // Extract the request. packetContentUnpack(data->data, "iss", &request, &sha256, &path); logWrite("[%s] [%s]\n\r", sha256, path); // Look up the entry and compare SHA256. - dbFileSha256Get(path, buffer, FILE_VIRTUAL_PATH_MAX); - if (strcasecmp(sha256, buffer) == 0) { + dbFileSha256Get(path, shaInDB, FILE_VIRTUAL_PATH_MAX); + if (strcasecmp(sha256, shaInDB) == 0) { logWrite("File is current. Sending FILE_RESPONSE_OKAY.\n\r"); // File is already current on client. - temp = FILE_RESPONSE_OKAY; - packetData = packetContentPack(&length, "i", temp); + temp = FILE_RESPONSE_OKAY; + packetData = packetContentPack(&length, "i", temp); encoded.control = PACKET_CONTROL_DAT; encoded.packetType = PACKET_TYPE_FILE_RESPONSE; encoded.channel = data->channel; @@ -103,21 +110,32 @@ static void clientApiFileRequestCheck(ClientThreadT *client, PacketDecodeDataT * logWrite("File needs updated.\n\r"); // Get real path. dbFileRealPathGet(path, buffer, FILE_VIRTUAL_PATH_MAX); - snprintf(buffer2, 1024, "%s%s", __settingsFile, buffer); + snprintf(buffer2, 2048, "%s%s", __settingsFile, buffer); // Open file & get file size. client->handle = fopen(buffer2, "rb"); if (!client->handle) { + logWrite("Unable to open [%s]\n", buffer2); // ***TODO*** Handle error + return; } fseek(client->handle, 0, SEEK_END); client->fileSize = ftell(client->handle); fseek(client->handle, 0, SEEK_SET); - // Start sending new file to client. - packetData = (char *)malloc(PACKET_MAX); - if (!packetData) { - // ***TODO*** Handle error - } logWrite("Size is %d\n\r", client->fileSize); + + // Send file metadata to start transfer + logWrite("Sending FILE_RESPONSE_SEND.\n\r"); + temp = FILE_RESPONSE_SEND; + packetData = packetContentPack(&length, "iis", temp, client->fileSize, shaInDB); + encoded.control = PACKET_CONTROL_DAT; + encoded.packetType = PACKET_TYPE_FILE_RESPONSE; + encoded.channel = data->channel; + encoded.encrypt = 0; + packetEncode(client->packetThreadData, &encoded, packetData, length); + packetSend(client->packetThreadData, &encoded); + DEL(packetData); + + // Start sending actual file data. clientApiFileRequestNext(client, data); } } @@ -127,39 +145,36 @@ static void clientApiFileRequestNext(ClientThreadT *client, PacketDecodeDataT *d uint32_t temp = 0; PacketEncodeDataT encoded = { 0 }; uint16_t length = 0; - uint16_t dataSize = 0; char packetData[PACKET_MAX]; // Do we have an open file? if (client->handle) { - logWrite("Sending FILE_RESPONSE_SEND.\n\r"); + logWrite("Sending FILE_RESPONSE_DATA.\n\r"); // Add response type. - temp = FILE_RESPONSE_SEND; - memcpy(packetData, &temp, sizeof(int32_t)); - - // Add file length. - memcpy(&packetData[4], &client->fileSize, sizeof(int32_t)); + temp = FILE_RESPONSE_DATA; + memcpy(&packetData[0], &temp, sizeof(int32_t)); // Add file data. - dataSize = client->fileSize - ftell(client->handle); - if (dataSize > PACKET_MAX - 8) { + // ***TODO*** We can't send quite a full packet for some reason. + length = client->fileSize - ftell(client->handle); + if (length > PACKET_MAX - 14) { // File is larger than a packet size. - dataSize = PACKET_MAX - 8; // 8 for two integers of response data. - fread(&packetData[8], dataSize, 1, client->handle); + length = PACKET_MAX - 14; // 4 for integer of response type. 10 more because a full PACKET_MAX causes issues. + fread(&packetData[4], length, 1, client->handle); } else { // File remains will fit in this packet. - fread(&packetData[8], dataSize, 1, client->handle); + fread(&packetData[4], length, 1, client->handle); fclose(client->handle); client->handle = NULL; } - logWrite("Sending %d bytes\n\r", dataSize); + logWrite("Sending %d bytes of file\n\r", length); // Build packet. encoded.control = PACKET_CONTROL_DAT; encoded.packetType = PACKET_TYPE_FILE_RESPONSE; encoded.channel = data->channel; encoded.encrypt = 0; - packetEncode(client->packetThreadData, &encoded, packetData, length); + packetEncode(client->packetThreadData, &encoded, packetData, length + 4); // Send it. packetSend(client->packetThreadData, &encoded); } diff --git a/server/src/database.c b/server/src/database.c index bc9bd79..1b3b259 100644 --- a/server/src/database.c +++ b/server/src/database.c @@ -26,6 +26,9 @@ #include "database.h" +// ***TODO*** Left/Right trim whitespace from returned strings? + + #define STATEMENT_MAX 2048 diff --git a/server/src/server.c b/server/src/server.c index 2ed98a9..7e67d7a 100644 --- a/server/src/server.c +++ b/server/src/server.c @@ -153,6 +153,8 @@ void *serverThread(void *data) { client->pingLow = 9999; // User State Stuff. client->authenticated = 0; + client->handle = NULL; + client->fileSize = 0; // Keep our client in the peer data for later. event.peer->data = (void *)client; // Make new thread for this client. diff --git a/shared/packets.h b/shared/packets.h index 83c69e6..52b5f91 100644 --- a/shared/packets.h +++ b/shared/packets.h @@ -64,7 +64,9 @@ typedef enum PacketTypeE { * Client -> PACKET_TYPE_FILE_REQUEST FILE_REQUEST_CHECK SHA256 VPATH * Server -> PACKET_TYPE_FILE_RESPONSE FILE_RESPONSE_OKAY * - or - - * Server -> PACKET_TYPE_FILE_RESPONSE FILE_RESPONSE_SEND (4 bytes length) (data...) + * Server -> PACKET_TYPE_FILE_RESPONSE FILE_RESPONSE_SEND (4 bytes length) SHA256 + * + * Server -> PACKET_TYPE_FILE_RESPONSE FILE_RESPONSE_DATA (data...) * Client -> PACKET_TYPE_FILE_REQUEST FILE_REQUEST_NEXT * (Repeat until received data matches the length.) */ @@ -76,6 +78,7 @@ typedef enum FileRequestsE { typedef enum FileResponseE { FILE_RESPONSE_UNKNOWN = 0, + FILE_RESPONSE_DATA, FILE_RESPONSE_OKAY, FILE_RESPONSE_SEND } FileResponseT;