// calogArchive.c -- calog compression + archive library (see calogArchive.h), built on libarchive. // // Two capabilities: one-shot raw codecs (compress/decompress a whole buffer in memory) and archive // read/write over the shared typed handle table. All data is binary-safe; a per-transfer cap // (ARC_MAX) guards against a decompression bomb. Every native is inline (runs on the calling // context's thread). libarchive needs no global init; the process-wide lib struct only owns a // refcounted handle table, exactly like calogNet. #include "calogArchive.h" #include "calogHandle.h" #include "calogInternal.h" #include #include #include #include #include #include #include // Handle kinds, distinct across the registry so a read handle passed to a write native (or vice // versa) fails to resolve. #define ARC_TYPE_READ 1u #define ARC_TYPE_WRITE 2u // Upper bound on a single one-shot buffer / entry read, so a hostile input cannot exhaust memory. #define ARC_MAX (64 * 1024 * 1024) // A growable byte buffer backing the in-memory write sink. typedef struct ArcBufT { char *data; size_t len; size_t cap; } ArcBufT; // A read handle: the libarchive reader plus, for an in-memory source, the owned copy it streams // from (libarchive does not copy the source buffer, so it must outlive the reader). typedef struct ArcReadT { struct archive *archive; void *memory; } ArcReadT; // A write handle: the libarchive writer feeding the growable in-memory sink. typedef struct ArcWriteT { struct archive *archive; ArcBufT sink; } ArcWriteT; // Process-wide state shared by every runtime that registers the natives. typedef struct ArcLibT { CalogHandleTableT *handles; int32_t refCount; } ArcLibT; typedef struct ArcNativeT { const char *name; CalogNativeFnT fn; } ArcNativeT; static pthread_mutex_t gArcMutex = PTHREAD_MUTEX_INITIALIZER; static ArcLibT *gArcLib = NULL; static int32_t arcBufAppend(ArcBufT *buffer, const void *bytes, size_t length); static void arcCloser(uint32_t type, void *resource); static int32_t arcCompress(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcDecompress(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcEntryTypeName(mode_t filetype, const char **out); static CalogValueT *arcMapField(CalogAggT *map, const char *name); static int32_t arcReadClose(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcReadData(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcReadNext(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcReadOpen(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcReadOpenFile(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcStore(ArcLibT *lib, uint32_t type, void *resource, CalogValueT *result); static la_ssize_t arcWriteSink(struct archive *archive, void *client, const void *buffer, size_t length); static int32_t arcWriteEntry(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcWriteFinish(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static int32_t arcWriteOpen(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData); static const ArcNativeT gArcNatives[] = { { "compress", arcCompress }, { "decompress", arcDecompress }, { "archiveReadOpen", arcReadOpen }, { "archiveReadOpenFile", arcReadOpenFile }, { "archiveReadNext", arcReadNext }, { "archiveReadData", arcReadData }, { "archiveReadClose", arcReadClose }, { "archiveWriteOpen", arcWriteOpen }, { "archiveWriteEntry", arcWriteEntry }, { "archiveWriteFinish", arcWriteFinish }, }; int32_t calogArchiveRegister(CalogT *calog) { ArcLibT *lib; int32_t status; size_t index; pthread_mutex_lock(&gArcMutex); if (gArcLib == NULL) { ArcLibT *created; created = (ArcLibT *)calloc(1, sizeof(*created)); if (created == NULL) { pthread_mutex_unlock(&gArcMutex); return calogErrOomE; } created->handles = calogHandleTableCreate(); if (created->handles == NULL) { free(created); pthread_mutex_unlock(&gArcMutex); return calogErrOomE; } gArcLib = created; } gArcLib->refCount++; lib = gArcLib; pthread_mutex_unlock(&gArcMutex); status = calogOkE; for (index = 0; index < sizeof(gArcNatives) / sizeof(gArcNatives[0]); index++) { status = calogRegisterInline(calog, gArcNatives[index].name, gArcNatives[index].fn, lib); if (status != calogOkE) { break; } } if (status != calogOkE) { calogArchiveShutdown(); return status; } return calogAtDestroy(calog, calogArchiveShutdown, calogDestroyAfterContextsE); } void calogArchiveShutdown(void) { pthread_mutex_lock(&gArcMutex); if (gArcLib == NULL) { pthread_mutex_unlock(&gArcMutex); return; } gArcLib->refCount--; if (gArcLib->refCount <= 0) { calogHandleTableDestroy(gArcLib->handles, arcCloser); free(gArcLib); gArcLib = NULL; } pthread_mutex_unlock(&gArcMutex); } static int32_t arcBufAppend(ArcBufT *buffer, const void *bytes, size_t length) { if (buffer->len + length > buffer->cap) { size_t wanted; char *grown; wanted = buffer->cap ? buffer->cap * 2 : 4096; while (wanted < buffer->len + length) { wanted *= 2; } grown = (char *)realloc(buffer->data, wanted); if (grown == NULL) { return calogErrOomE; } buffer->data = grown; buffer->cap = wanted; } memcpy(buffer->data + buffer->len, bytes, length); buffer->len += length; return calogOkE; } static void arcCloser(uint32_t type, void *resource) { switch (type) { case ARC_TYPE_READ: { ArcReadT *reader; reader = (ArcReadT *)resource; if (reader->archive != NULL) { archive_read_free(reader->archive); } free(reader->memory); free(reader); break; } case ARC_TYPE_WRITE: { ArcWriteT *writer; writer = (ArcWriteT *)resource; if (writer->archive != NULL) { archive_write_free(writer->archive); } free(writer->sink.data); free(writer); break; } default: break; } } static int32_t arcCompress(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { struct archive *archive; struct archive_entry *entry; ArcBufT sink; const char *filter; int32_t status; (void)userData; calogValueNil(result); if (argCount < 2 || argCount > 3 || args[0].type != calogStringE || args[1].type != calogStringE) { return calogFail(result, calogErrArgE, "compress expects (data, filter [, level])"); } if (argCount == 3 && args[2].type != calogIntE) { return calogFail(result, calogErrArgE, "compress: level must be an integer"); } filter = args[1].as.s.bytes; archive = archive_write_new(); if (archive == NULL) { return calogFail(result, calogErrOomE, "compress: out of memory"); } archive_write_set_format_raw(archive); // No block padding for a one-shot codec: the output is exactly the filtered stream, so a // reader (e.g. zstd/lz4) does not mistake trailing NUL padding for a corrupt second frame. archive_write_set_bytes_per_block(archive, 1); archive_write_set_bytes_in_last_block(archive, 1); if (strcmp(filter, "none") != 0 && archive_write_add_filter_by_name(archive, filter) != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, archive_error_string(archive)); archive_write_free(archive); return status; } if (argCount == 3) { char option[64]; snprintf(option, sizeof(option), "%s:compression-level=%lld", filter, (long long)args[2].as.i); archive_write_set_options(archive, option); // best effort; unknown option is ignored } memset(&sink, 0, sizeof(sink)); if (archive_write_open2(archive, &sink, NULL, arcWriteSink, NULL, NULL) != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, archive_error_string(archive)); archive_write_free(archive); free(sink.data); return status; } entry = archive_entry_new(); if (entry == NULL) { archive_write_free(archive); free(sink.data); return calogFail(result, calogErrOomE, "compress: out of memory"); } archive_entry_set_pathname(entry, "data"); archive_entry_set_filetype(entry, AE_IFREG); archive_entry_set_size(entry, args[0].as.s.length); status = calogOkE; if (archive_write_header(archive, entry) != ARCHIVE_OK || archive_write_data(archive, args[0].as.s.bytes, (size_t)args[0].as.s.length) < 0) { status = calogFail(result, calogErrArgE, archive_error_string(archive)); } archive_entry_free(entry); if (status == calogOkE && archive_write_close(archive) != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, archive_error_string(archive)); } archive_write_free(archive); if (status == calogOkE) { status = calogValueString(result, sink.data ? sink.data : "", (int64_t)sink.len); } free(sink.data); return status; } static int32_t arcDecompress(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { struct archive *archive; struct archive_entry *entry; ArcBufT out; int code; int32_t status; (void)userData; calogValueNil(result); if (argCount < 1 || argCount > 2 || args[0].type != calogStringE) { return calogFail(result, calogErrArgE, "decompress expects (data [, filter])"); } archive = archive_read_new(); if (archive == NULL) { return calogFail(result, calogErrOomE, "decompress: out of memory"); } archive_read_support_filter_all(archive); archive_read_support_format_raw(archive); if (archive_read_open_memory(archive, args[0].as.s.bytes, (size_t)args[0].as.s.length) != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, archive_error_string(archive)); archive_read_free(archive); return status; } code = archive_read_next_header(archive, &entry); if (code != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, code == ARCHIVE_EOF ? "decompress: empty input" : archive_error_string(archive)); archive_read_free(archive); return status; } memset(&out, 0, sizeof(out)); status = calogOkE; for (;;) { char chunk[65536]; la_ssize_t got; got = archive_read_data(archive, chunk, sizeof(chunk)); if (got < 0) { status = calogFail(result, calogErrArgE, archive_error_string(archive)); break; } if (got == 0) { break; } if (out.len + (size_t)got > ARC_MAX) { status = calogFail(result, calogErrRangeE, "decompress: output exceeds the size cap"); break; } if (arcBufAppend(&out, chunk, (size_t)got) != calogOkE) { status = calogFail(result, calogErrOomE, "decompress: out of memory"); break; } } archive_read_free(archive); if (status == calogOkE) { status = calogValueString(result, out.data ? out.data : "", (int64_t)out.len); } free(out.data); return status; } static int32_t arcEntryTypeName(mode_t filetype, const char **out) { switch (filetype) { case AE_IFREG: *out = "file"; return calogOkE; case AE_IFDIR: *out = "dir"; return calogOkE; case AE_IFLNK: *out = "symlink"; return calogOkE; case AE_IFSOCK: *out = "socket"; return calogOkE; case AE_IFCHR: *out = "chardev"; return calogOkE; case AE_IFBLK: *out = "blockdev"; return calogOkE; case AE_IFIFO: *out = "fifo"; return calogOkE; default: *out = "unknown"; return calogOkE; } } // Look up a string-keyed field in a map (borrowed pointer into the map, or NULL). static CalogValueT *arcMapField(CalogAggT *map, const char *name) { CalogValueT key; CalogValueT *field; if (calogValueString(&key, name, (int64_t)strlen(name)) != calogOkE) { return NULL; } field = calogAggGet(map, &key); calogValueFree(&key); return field; } static int32_t arcReadClose(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcReadT *reader; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount != 1 || args[0].type != calogIntE) { return calogFail(result, calogErrArgE, "archiveReadClose expects (handle)"); } reader = (ArcReadT *)calogHandleRemove(lib->handles, args[0].as.i, ARC_TYPE_READ); if (reader == NULL) { return calogFail(result, calogErrArgE, "archiveReadClose: invalid handle"); } arcCloser(ARC_TYPE_READ, reader); return calogOkE; } static int32_t arcReadData(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcReadT *reader; ArcBufT out; int32_t status; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount != 1 || args[0].type != calogIntE) { return calogFail(result, calogErrArgE, "archiveReadData expects (handle)"); } reader = (ArcReadT *)calogHandleGet(lib->handles, args[0].as.i, ARC_TYPE_READ); if (reader == NULL) { return calogFail(result, calogErrArgE, "archiveReadData: invalid handle"); } memset(&out, 0, sizeof(out)); status = calogOkE; for (;;) { char chunk[65536]; la_ssize_t got; got = archive_read_data(reader->archive, chunk, sizeof(chunk)); if (got < 0) { status = calogFail(result, calogErrArgE, archive_error_string(reader->archive)); break; } if (got == 0) { break; } if (out.len + (size_t)got > ARC_MAX) { status = calogFail(result, calogErrRangeE, "archiveReadData: entry exceeds the size cap"); break; } if (arcBufAppend(&out, chunk, (size_t)got) != calogOkE) { status = calogFail(result, calogErrOomE, "archiveReadData: out of memory"); break; } } if (status == calogOkE) { status = calogValueString(result, out.data ? out.data : "", (int64_t)out.len); } free(out.data); return status; } static int32_t arcReadNext(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcReadT *reader; struct archive_entry *entry; CalogAggT *map; const char *name; const char *typeName; int code; int32_t status; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount != 1 || args[0].type != calogIntE) { return calogFail(result, calogErrArgE, "archiveReadNext expects (handle)"); } reader = (ArcReadT *)calogHandleGet(lib->handles, args[0].as.i, ARC_TYPE_READ); if (reader == NULL) { return calogFail(result, calogErrArgE, "archiveReadNext: invalid handle"); } code = archive_read_next_header(reader->archive, &entry); if (code == ARCHIVE_EOF) { return calogOkE; // nil signals the end of the archive } if (code != ARCHIVE_OK && code != ARCHIVE_WARN) { return calogFail(result, calogErrArgE, archive_error_string(reader->archive)); } status = calogAggCreate(&map, calogMapE); if (status != calogOkE) { return calogFail(result, status, "archiveReadNext: out of memory"); } name = archive_entry_pathname(entry); arcEntryTypeName(archive_entry_filetype(entry), &typeName); status = calogMapSetStr(map, "name", name ? name : "", name ? (int64_t)strlen(name) : 0); if (status == calogOkE) { status = calogMapSetInt(map, "size", (int64_t)archive_entry_size(entry)); } if (status == calogOkE) { status = calogMapSetInt(map, "mode", (int64_t)archive_entry_perm(entry)); } if (status == calogOkE) { status = calogMapSetInt(map, "mtime", (int64_t)archive_entry_mtime(entry)); } if (status == calogOkE) { status = calogMapSetStr(map, "type", typeName, (int64_t)strlen(typeName)); } if (status == calogOkE && archive_entry_filetype(entry) == AE_IFLNK) { const char *target; target = archive_entry_symlink(entry); if (target != NULL) { status = calogMapSetStr(map, "linkname", target, (int64_t)strlen(target)); } } if (status != calogOkE) { calogAggFree(map); return calogFail(result, status, "archiveReadNext: failed to build the entry"); } calogValueAgg(result, map); return calogOkE; } static int32_t arcReadOpen(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcReadT *reader; void *copy; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount != 1 || args[0].type != calogStringE) { return calogFail(result, calogErrArgE, "archiveReadOpen expects (bytes)"); } reader = (ArcReadT *)calloc(1, sizeof(*reader)); if (reader == NULL) { return calogFail(result, calogErrOomE, "archiveReadOpen: out of memory"); } // libarchive streams the source lazily and does NOT copy it, so keep an owned copy alive. copy = malloc(args[0].as.s.length > 0 ? (size_t)args[0].as.s.length : 1); if (copy == NULL) { free(reader); return calogFail(result, calogErrOomE, "archiveReadOpen: out of memory"); } memcpy(copy, args[0].as.s.bytes, (size_t)args[0].as.s.length); reader->memory = copy; reader->archive = archive_read_new(); if (reader->archive == NULL) { free(copy); free(reader); return calogFail(result, calogErrOomE, "archiveReadOpen: out of memory"); } archive_read_support_filter_all(reader->archive); archive_read_support_format_all(reader->archive); if (archive_read_open_memory(reader->archive, copy, (size_t)args[0].as.s.length) != ARCHIVE_OK) { int32_t status; status = calogFail(result, calogErrArgE, archive_error_string(reader->archive)); arcCloser(ARC_TYPE_READ, reader); return status; } return arcStore(lib, ARC_TYPE_READ, reader, result); } static int32_t arcReadOpenFile(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcReadT *reader; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount != 1 || args[0].type != calogStringE) { return calogFail(result, calogErrArgE, "archiveReadOpenFile expects (path)"); } reader = (ArcReadT *)calloc(1, sizeof(*reader)); if (reader == NULL) { return calogFail(result, calogErrOomE, "archiveReadOpenFile: out of memory"); } reader->archive = archive_read_new(); if (reader->archive == NULL) { free(reader); return calogFail(result, calogErrOomE, "archiveReadOpenFile: out of memory"); } archive_read_support_filter_all(reader->archive); archive_read_support_format_all(reader->archive); if (archive_read_open_filename(reader->archive, args[0].as.s.bytes, 65536) != ARCHIVE_OK) { int32_t status; status = calogFail(result, calogErrArgE, archive_error_string(reader->archive)); arcCloser(ARC_TYPE_READ, reader); return status; } return arcStore(lib, ARC_TYPE_READ, reader, result); } // Wrap a resource in a handle-table entry, transferring ownership. On failure the resource is // freed via arcCloser. Sets result to the new integer handle on success. static int32_t arcStore(ArcLibT *lib, uint32_t type, void *resource, CalogValueT *result) { int64_t handle; handle = calogHandleAdd(lib->handles, type, resource); if (handle == 0) { arcCloser(type, resource); return calogFail(result, calogErrOomE, "out of memory"); } calogValueInt(result, handle); return calogOkE; } static la_ssize_t arcWriteSink(struct archive *archive, void *client, const void *buffer, size_t length) { (void)archive; if (arcBufAppend((ArcBufT *)client, buffer, length) != calogOkE) { return -1; } return (la_ssize_t)length; } static int32_t arcWriteEntry(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcWriteT *writer; struct archive_entry *entry; int32_t status; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount < 3 || argCount > 4 || args[0].type != calogIntE || args[1].type != calogStringE || args[2].type != calogStringE) { return calogFail(result, calogErrArgE, "archiveWriteEntry expects (handle, name, data [, opts])"); } if (argCount == 4 && args[3].type != calogAggE) { return calogFail(result, calogErrArgE, "archiveWriteEntry: opts must be a map"); } writer = (ArcWriteT *)calogHandleGet(lib->handles, args[0].as.i, ARC_TYPE_WRITE); if (writer == NULL) { return calogFail(result, calogErrArgE, "archiveWriteEntry: invalid handle"); } entry = archive_entry_new(); if (entry == NULL) { return calogFail(result, calogErrOomE, "archiveWriteEntry: out of memory"); } archive_entry_set_pathname(entry, args[1].as.s.bytes); archive_entry_set_filetype(entry, AE_IFREG); archive_entry_set_perm(entry, 0644); archive_entry_set_size(entry, args[2].as.s.length); if (argCount == 4) { CalogValueT *field; field = arcMapField(args[3].as.agg,"mode"); if (field != NULL && field->type == calogIntE) { archive_entry_set_perm(entry, (mode_t)field->as.i); } field = arcMapField(args[3].as.agg,"mtime"); if (field != NULL && field->type == calogIntE) { archive_entry_set_mtime(entry, (time_t)field->as.i, 0); } field = arcMapField(args[3].as.agg,"type"); if (field != NULL && field->type == calogStringE && strcmp(field->as.s.bytes, "dir") == 0) { archive_entry_set_filetype(entry, AE_IFDIR); archive_entry_set_size(entry, 0); } } status = calogOkE; if (archive_write_header(writer->archive, entry) != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, archive_error_string(writer->archive)); } else if (args[2].as.s.length > 0 && archive_write_data(writer->archive, args[2].as.s.bytes, (size_t)args[2].as.s.length) < 0) { status = calogFail(result, calogErrArgE, archive_error_string(writer->archive)); } archive_entry_free(entry); return status; } static int32_t arcWriteFinish(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcWriteT *writer; int32_t status; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount != 1 || args[0].type != calogIntE) { return calogFail(result, calogErrArgE, "archiveWriteFinish expects (handle)"); } writer = (ArcWriteT *)calogHandleRemove(lib->handles, args[0].as.i, ARC_TYPE_WRITE); if (writer == NULL) { return calogFail(result, calogErrArgE, "archiveWriteFinish: invalid handle"); } status = calogOkE; if (archive_write_close(writer->archive) != ARCHIVE_OK) { status = calogFail(result, calogErrArgE, archive_error_string(writer->archive)); } if (status == calogOkE) { status = calogValueString(result, writer->sink.data ? writer->sink.data : "", (int64_t)writer->sink.len); } arcCloser(ARC_TYPE_WRITE, writer); return status; } static int32_t arcWriteOpen(CalogValueT *args, int32_t argCount, CalogValueT *result, void *userData) { ArcLibT *lib; ArcWriteT *writer; const char *format; const char *filter; lib = (ArcLibT *)userData; calogValueNil(result); if (argCount < 2 || argCount > 3 || args[0].type != calogStringE || args[1].type != calogStringE) { return calogFail(result, calogErrArgE, "archiveWriteOpen expects (format, filter [, level])"); } if (argCount == 3 && args[2].type != calogIntE) { return calogFail(result, calogErrArgE, "archiveWriteOpen: level must be an integer"); } // "tar" is the friendly name for pax-restricted, libarchive's portable modern default. format = strcmp(args[0].as.s.bytes, "tar") == 0 ? "paxr" : args[0].as.s.bytes; filter = args[1].as.s.bytes; writer = (ArcWriteT *)calloc(1, sizeof(*writer)); if (writer == NULL) { return calogFail(result, calogErrOomE, "archiveWriteOpen: out of memory"); } writer->archive = archive_write_new(); if (writer->archive == NULL) { free(writer); return calogFail(result, calogErrOomE, "archiveWriteOpen: out of memory"); } if (archive_write_set_format_by_name(writer->archive, format) != ARCHIVE_OK) { int32_t status; status = calogFail(result, calogErrArgE, archive_error_string(writer->archive)); arcCloser(ARC_TYPE_WRITE, writer); return status; } if (strcmp(filter, "none") != 0 && archive_write_add_filter_by_name(writer->archive, filter) != ARCHIVE_OK) { int32_t status; status = calogFail(result, calogErrArgE, archive_error_string(writer->archive)); arcCloser(ARC_TYPE_WRITE, writer); return status; } if (argCount == 3) { char option[64]; snprintf(option, sizeof(option), "%s:compression-level=%lld", filter, (long long)args[2].as.i); archive_write_set_options(writer->archive, option); // best effort } if (archive_write_open2(writer->archive, &writer->sink, NULL, arcWriteSink, NULL, NULL) != ARCHIVE_OK) { int32_t status; status = calogFail(result, calogErrArgE, archive_error_string(writer->archive)); arcCloser(ARC_TYPE_WRITE, writer); return status; } return arcStore(lib, ARC_TYPE_WRITE, writer, result); }