/** * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl * */ #define _GNU_SOURCE #include "fdleak.h" #include "vec.h" #include "putils.h" #include "fwg.h" #include "inodemap.h" #include "malloc.h" #include "sched.h" #include "stringset.h" #include "vector-of-strings.h" #include "version.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "common/spew.h" #include // Parallel tar. For parallel untar The main thread scans through the // tar file. If it finds a directory, it creates it. If it finds a // regular file, it mallocs a place to hold the contents of the file, // scans everything into the alloced object, and stores the alloced // object into a vector of items to work on. (The main thread also // handles symlinks: rationale symlinks seem to be uncommon, and it's // simpler this way.) // // Meanwhile, a bunch of threads grab items out of the vector. Each // item involves creating a file and writing the bytes into the file. // They grab a random item to reduce the chances that they are trying // to create files in the same directory. // // We cannot handle the case where the same file appears twice in the // tar file (presumably the semantics is that the second file "wins".) // We use a hash table to detect that this happens. The hash table // doesn't need a mutex since it's only accessed by the main thread. /* tar Header Block, from POSIX 1003.1-1990. */ /* POSIX header. */ struct posix_header { /* byte offset */ char name[100]; /* 0 */ char mode[8]; /* 100 */ char uid[8]; /* 108 */ char gid[8]; /* 116 */ char size[12]; /* 124 */ char mtime[12]; /* 136 */ char chksum[8]; /* 148 */ char typeflag[1]; /* 156 */ char linkname[100]; /* 157 */ char magic[6]; /* 257 */ char version[2]; /* 263 */ char uname[32]; /* 265 */ char gname[32]; /* 297 */ char devmajor[8]; /* 329 */ char devminor[8]; /* 337 */ char prefix[155]; /* 345 */ /* 500 */ char pad[12]; }; static void print_byte(char c) { if (isprint(c) && c != '\\') { fprintf(stderr, "%c", c); } else { fprintf(stderr, "\\%03o", c); } } static void print_bytes(const char *s, size_t len) { fprintf(stderr, "\""); for (size_t i = 0; i < len; i++) { print_byte(s[i]); } fprintf(stderr, "\""); } #define PRINT_BYTES(name) (fprintf(stderr, "\n %8s = ", #name), print_bytes(h->name, sizeof(h->name))) static void debug_print_posix_header(const struct posix_header *h) { fprintf(stderr, "posix header {"); PRINT_BYTES(name); PRINT_BYTES(mode); PRINT_BYTES(uid); PRINT_BYTES(gid); PRINT_BYTES(size); PRINT_BYTES(mtime); PRINT_BYTES(chksum); PRINT_BYTES(typeflag); PRINT_BYTES(linkname); PRINT_BYTES(magic); PRINT_BYTES(version); PRINT_BYTES(uname); PRINT_BYTES(gname); PRINT_BYTES(devmajor); PRINT_BYTES(devminor); PRINT_BYTES(prefix); PRINT_BYTES(pad); fprintf(stderr, "\n ... }\n"); } enum { default_memory_budget_MiB = 100 }; enum { MiB = 1024*1024 }; static size_t memory_budget_MiB = default_memory_budget_MiB; enum { max_name_length = 255 }; enum { DONE = 0, ERROR = 1, CONTINUE = 2, }; #define ROOT_UID 0 static int we_are_root; static int arg_preserve = 0; // default is 1 if we are root static int arg_gunzip = 0; static int arg_show_progress = 0; static const char *arg_named_input = NULL; static const char *arg_change_dir = NULL; pthread_cond_t fwcond = PTHREAD_COND_INITIALIZER; pthread_mutex_t fwmutex = PTHREAD_MUTEX_INITIALIZER; struct fwobject { char command; char *fullname; char *linkname; size_t size; size_t mode; size_t uid; size_t gid; size_t mtime; // If bytes!=NULL then the data is in bytes. char *bytes; // // Else it's in input_file, to be gotten by do_read(input_file, // size size_to_read), set *done and signal fwcond when done. FILE *input_file; size_t size_to_read; int *done; }; static struct fwobject *mk_fwobject(char command, const char *fullname, const char *linkname, size_t size, size_t mode, size_t uid, size_t gid, size_t mtime, char *bytes/*[size]*/) { struct fwobject *MALLOC(b); *b = (struct fwobject){.command = command, .fullname = fullname ? strdup(fullname) : NULL, .linkname = linkname ? strdup(linkname) : NULL, .size = size, .mode = mode, .uid = uid, .gid = gid, .mtime = mtime, .bytes = bytes, .input_file = NULL, .size_to_read = 0, .done = NULL}; return b; } static struct fwobject *mk_fwobject_read_from_input(char command, const char *fullname, size_t size, size_t mode, size_t uid, size_t gid, size_t mtime, FILE *input_file, size_t size_to_read, int *done) { struct fwobject *MALLOC(b); *b = (struct fwobject){.command = command, .fullname = fullname ? strdup(fullname) : NULL, .linkname = NULL, .size = size, .mode = mode, .uid = uid, .gid = gid, .mtime = mtime, .bytes = NULL, .input_file = input_file, .size_to_read = size_to_read, .done = done}; return b; } static magic_t mep_magic = "mep magic"; struct mep_extra { magic_t *magic; const char *fname; }; static int matches_excludes_predicate(const char *pattern, void *mepv) { struct mep_extra *mep = mepv; assert(mep->magic == &mep_magic); spew(SPEW_DEBUG, "matching %s to %s", mep->fname, pattern); const char *str = mep->fname; while (*str) { spew(SPEW_DEBUG, "str=%s", str); const char *end = index(str, '/'); if (end == NULL) { spew(SPEW_DEBUG, "matching %s to %s", str, pattern); int r = fnmatch(pattern, str, 0); spew(SPEW_DEBUG, "%d", !r); return !r; } else if (strcmp(end, str) == 0) { str = end+1; spew(SPEW_DEBUG, "str=%s end=%s", str,end); } else { char path[256]; assert(end > str); assert((size_t)(end-str) < sizeof(path) - 1); memcpy(path, str, (size_t)(end-str)); path[end-str] = 0; spew(SPEW_DEBUG, "matching %s to %s", path, pattern); if (fnmatch(pattern, path, 0) == 0) { spew(SPEW_DEBUG, "%s matches %s (original %s)", path, pattern, mep->fname); return 1; } str = end+1; spew(SPEW_DEBUG, "now str=%s", str); } } spew(SPEW_DEBUG, "nope"); return 0; } int some_errors = 0; static struct vector_of_strings *excludes; static int matches_excludes(const char *fname) { struct mep_extra mep = {&mep_magic, fname}; return vector_of_strings_any(excludes, matches_excludes_predicate, &mep); } static struct fwg *fwg; static void pinit(void) { fwg = mk_fwg(memory_budget_MiB * 1024ul * 1024ul); }; static void pdestroy(void) { fwg = fwg_destroy(fwg); } static size_t smin(size_t a, size_t b) { if (a < b) return a; else return b; } /* * The function checks the stat of the files that already exist. * It then compares the type of the existing file with the new file * that needs to be created at the same path. In case of mismatch * the existing file is deleted. If the existing file type is * directory, it deletes the subtree as well. */ static int check_and_delete_conflicts(char typeflag, char *fullname) { struct stat filestat; char tempname[600]; snprintf(tempname, sizeof(tempname), "%s", fullname); if (tempname[strlen(tempname) - 1] == '/') { tempname[strlen(tempname) - 1] = 0; } int ret = stat(tempname, &filestat); if (ret == 0) { if (S_ISDIR(filestat.st_mode) && typeflag != '5') { /* * Found a directory at the path where a new type * of file is expected. Deleting along with subtree. */ delete_dir_recursively(tempname); } else if ((S_ISREG(filestat.st_mode) && typeflag != '0') || (S_ISLNK(filestat.st_mode) && typeflag != '2') || (S_ISFIFO(filestat.st_mode) && typeflag != '6')) { do_unlink(tempname); } } return 0; } static void process_fwobject(size_t threadn, struct fwobject *b, struct fwgnode *handle, int retry) { int ret; spew(SPEW_DEBUG, "cmd='%c' %s handle=%p", b->command, b->fullname, handle); switch (b->command) { case '0': { // create file spew(SPEW_DEBUG, "%lu: parallel (small) regular file \"%s\"", threadn, b->fullname); chmod(b->fullname, 0700); FILE *of = do_creat(b->fullname); if (of == NULL && retry == 1) { check_and_delete_conflicts(b->command, b->fullname); process_fwobject(threadn, b, handle, 0); return; } if (b->bytes) { assert(!b->input_file && !b->size_to_read && !b->done); do_write(of, b->bytes, b->size); } else { spew(SPEW_DEBUG, "Immediate read of %lu bytes into %lu bytes", b->size_to_read, b->size); assert(b->input_file && b->size_to_read && b->done); assert(!*b->done); size_t bufsize = 16ul * 1024ul * 1024ul; char *buf = malloc(bufsize); size_t remaining_to_read = b->size_to_read; size_t remaining_to_write = b->size; while (remaining_to_read) { assert(remaining_to_write <= remaining_to_read); size_t read_now = smin(remaining_to_read, bufsize); size_t r = fread(buf, 1, read_now, b->input_file); assert(r <= read_now); remaining_to_read -= read_now; if (r < read_now) { if (feof(b->input_file)) { fprintf(stderr, "read from %s encountered premature end of file", b->fullname); spew(SPEW_ERROR, "read from %s encountered premature end of file", b->fullname); } else { assert(ferror(b->input_file)); fprintf(stderr, "read from %s failed with error: %d\n", b->fullname, ferror(b->input_file)); spew(SPEW_ERROR, "read from %s failed with error: %d\n", b->fullname, ferror(b->input_file)); } exit(1); } else { size_t write_now = smin(remaining_to_write, read_now); size_t r2 = fwrite(buf, 1, write_now, of); assert(r2 == write_now); remaining_to_write -= write_now; } } FREE(buf); } { int r __attribute__((unused)) = fchmod(fileno(of), (mode_t)b->mode); // Don't check r. Either it worked or it didn't. } if (arg_preserve) { int r __attribute__((unused)) = fchown(fileno(of), (uid_t)b->uid, (gid_t)b->gid); // Don't check r. Either it worked or it didn't. Gnu // tar will try doing fchownat as a backup. Maybe // we'll have to do that too... } do_fclose(of); { struct timespec times[2] = {{.tv_nsec=UTIME_OMIT}, {.tv_sec = (time_t)b->mtime, .tv_nsec=0}}; int r = utimensat(AT_FDCWD, b->fullname, times, AT_SYMLINK_NOFOLLOW); if (r != 0) { fprintf(stderr, "utimensat failed on %s errno=%d (%s)\n", b->fullname, errno, strerror(errno)); spew(SPEW_ERROR, "utimensat failed on %s errno=%d (%s)", b->fullname, errno, strerror(errno)); __sync_add_and_fetch(&some_errors, 1); } } if (!b->bytes) { *b->done = 1; spew(SPEW_DEBUG, "Finished immediate read"); pthread_mutex_lock(&fwmutex); pthread_cond_signal(&fwcond); pthread_mutex_unlock(&fwmutex); } spew(SPEW_DEBUG, "finished %s", b->fullname); FREE(b->fullname); assert(!b->linkname); FREE(b->bytes); FREE(b); break; } case '1': { assert(b->fullname && b->fullname[0]); assert(b->linkname && b->linkname[0]); ret = do_link(b->linkname, b->fullname); if (ret != 0 && retry == 1) { check_and_delete_conflicts(b->command, b->fullname); process_fwobject(threadn, b, handle, 0); return; } FREE(b->linkname); FREE(b->fullname); FREE(b); break; } case '2': { ret = do_symlink(b->linkname, b->fullname); if (ret != 0 && errno == EEXIST && retry == 1) { check_and_delete_conflicts(b->command, b->fullname); process_fwobject(threadn, b, handle, 0); return; } { struct timespec times[2] = {{.tv_nsec=UTIME_OMIT}, {.tv_sec = (time_t)b->mtime, .tv_nsec=0}}; int r = utimensat(AT_FDCWD, b->fullname, times, AT_SYMLINK_NOFOLLOW); assert(r == 0); } FREE(b->linkname); FREE(b->fullname); FREE(b); break; } case '5': { ret = do_mkdir(b->fullname); if (ret != 0 && retry == 1) { check_and_delete_conflicts(b->command, b->fullname); process_fwobject(threadn, b, handle, 0); return; } FREE(b->fullname); assert(!b->linkname); FREE(b); break; } case '6': { ret = do_mkfifo(b->fullname, (mode_t)b->mode); if (ret != 0 && retry == 1) { check_and_delete_conflicts(b->command, b->fullname); process_fwobject(threadn, b, handle, 0); return; } FREE(b->fullname); assert(!b->linkname); FREE(b); break; } default: { fprintf(stderr, "Unknown command %c\n", b->command); spew(SPEW_DEBUG, "Unknown command %c", b->command); assert(0); } } spew(SPEW_DEBUG, "thread %lu finishing an fwobject", threadn); fwg_finish_node(fwg, handle); spew(SPEW_DEBUG, "%lu: next", threadn); } static void process_fwobjects(size_t threadn) { while (1) { struct fwobject *b; struct fwgnode *handle; int r = fwg_get_ready_node(fwg, &b, &handle); if (r != 0) { return; } assert(b); process_fwobject(threadn, b, handle, 1); } } static void* fwobjects_runner(void *arg) { size_t *ip = arg; process_fwobjects(*ip); return NULL; } enum { default_n_threads = 32 }; static size_t n_threads = default_n_threads; static pthread_t *threads; static size_t *threadid; static void start_runners(void) { MALLOC_N(threads, n_threads); MALLOC_N(threadid, n_threads); for (size_t i = 0; i < n_threads; i++) { threadid[i]=i; pthread_create(&threads[i], NULL, fwobjects_runner, &threadid[i]); } } static void runners_finish(void) { // After the last enqueue, call sfq_finish_up to wait for all the work to finish. for (size_t i = 0; i < n_threads; i++) { void *v; pthread_join(threads[i], &v); assert(v == NULL); } FREE(threads); FREE(threadid); } static size_t parse_octal_number(char *field, size_t sizeof_field) { assert(sizeof_field > 0); if (*field == '\200') { size_t result = 0; for (size_t i = 1; i < sizeof_field; i++) { result = (result * 256) + ((unsigned char *)field)[i]; } return result; } else { assert(0 == (*(unsigned char*)field & 0x80)); errno = 0; char *end; size_t result = strtoul(field, &end, 8); assert(errno == 0); assert(*end == 0 || *end==' '); assert(end < field+sizeof_field); return result; } } static const char *cmd; static char tarmode = ' '; void help(int exitcode) { spew(SPEW_DEBUG, "exitcode %d ", exitcode); FILE *out = exitcode ? stderr : stdout; fprintf(out, "Usage: %s [TAROPTION] [OPTION] FILE...\n", cmd); fprintf(out, "%s is a parallel implementation of tar. It can create and extract\n", cmd); fprintf(out, "tarballs in parallel, which can sometimes provide improved performance.\n"); fprintf(out, " TAROPTION is only the single-letter tar options with no prefix.\n"); fprintf(out, " TAROPTION can be\n"); fprintf(out, " p Preserve permissions (default for superuser).\n"); fprintf(out, " c Create a tarball. Specify FILE(s).\n"); fprintf(out, " x Extract files. Don't specify FILE(s). You need exactly one of x or c.\n"); fprintf(out, " z Filter the archive through gzip(1).\n"); fprintf(out, " f The next argument names the archive (the input tarball for x, the output for c). A dash `-' means stdin/stdout. Without f, the default is stdin/stdout.\n"); fprintf(out, " OPTION can be\n"); fprintf(out, " -C DIR Change to directory DIR\n"); fprintf(out, " -h, --help Print help.\n"); fprintf(out, " -P P (capital P) use P-fold parallelism (default %d threads).\n", default_n_threads); fprintf(out, " -m M Set memory budget to M mebibytes (default %d MiB).\n", default_memory_budget_MiB); fprintf(out, " --exclude=PATTERN Exclude patterns matching PATTERN, a glob(3)-style wildcard pattern.\n"); fprintf(out, " -p, --preserve-permissions, --same-permissions\n"); fprintf(out, " extract information about file permissions (default for superuser)\n"); fprintf(out, " --version Print program version.\n"); fprintf(out, " --history Print a brief history of the versions.\n"); fprintf(out, " --progress Monitor progress of untarring.\n"); fprintf(out, "This is version %s gitrev %s compiled %s %s.\n", VERSION, gitrev, __DATE__, __TIME__); fprintf(out, "Report bugs to parallel-tools-support_ww_grp@oracle.com\n"); } static int prefix_matches(const char *prefix, const char *string, char const **suffix) { while (*prefix == *string) { spew(SPEW_DEBUG, "prefix %s string %s", prefix, string); prefix++; string++; } if (*prefix == 0) { *suffix = string; spew(SPEW_DEBUG, "prefix %s suffix %s string %s", prefix, *suffix, string); return 1; } else { return 0; } } static int parse_args(int argc, char *argv[], struct vector_of_strings *vs) { assert(argc > 0); cmd = argv[0]; argc--; argv++; int at_first_arg = 1; char const *suffix; while (argc) { spew(SPEW_DEBUG, "at_first=%d arg parse = %s", at_first_arg, argv[0]); if (strcmp(argv[0], "-P") == 0) { argc--; argv++; if (argc == 0) { help(1); return ERROR; } n_threads = parse_number_or_help(argv[0]); if (n_threads == 0) { // something is wrong should've got a value return ERROR; } spew(SPEW_DEBUG, "number of threads = %ld", n_threads); } else if (strcmp(argv[0], "-m") == 0) { argc--; argv++; if (argc == 0) { help(1); return ERROR; } memory_budget_MiB = parse_number_or_help(argv[0]); if (memory_budget_MiB == 0) { // something is wrong should've got a value return ERROR; } spew(SPEW_DEBUG, "memory budget (MiB) = %ld", memory_budget_MiB); } else if (strcmp(argv[0], "-h") == 0 || strcmp(argv[0], "--help") == 0) { help(0); return DONE; } else if (strcmp(argv[0], "--version") == 0) { printf("%s %s\n" "Copyright (C) 2019 Oracle.\n" "Written by Bradley C. Kuszmaul\n", cmd, VERSION); return DONE; } else if (strcmp(argv[0], "--history") == 0) { fprintf(stderr, "%s", VERSION_HISTORY); return DONE; } else if (strcmp(argv[0], "-C") == 0) { argc--; argv++; if (argc == 0) { fprintf(stderr, "Need a DIR argument for -C\n"); spew(SPEW_ERROR, "Need a DIR argument for -C"); help(1); return ERROR; } arg_change_dir = argv[0]; } else if (strcmp(argv[0], "--progress") == 0) { arg_show_progress = 1; } else if (strcmp(argv[0], "--exclude") == 0) { argc--; argv++; if (argc == 0) { fprintf(stderr, "Need a PATTERN argument for --exclude\n"); spew(SPEW_ERROR, "Need a PATTERN argument for --exclude"); help(1); return ERROR; } spew(SPEW_DEBUG, "excluding %s", argv[0]); vector_of_strings_push(excludes, argv[0]); } else if (strcmp(argv[0], "-p") == 0 || strcmp(argv[0], "--preserve-permissions") == 0 || strcmp(argv[0], "--same-permissions") == 0) { arg_preserve = 1; } else if (prefix_matches("--exclude=", argv[0], &suffix)) { spew(SPEW_DEBUG, "excluding %s suffix %s", argv[0], suffix); vector_of_strings_push(excludes, suffix); } else if (at_first_arg && argv[0][0] != '-') { // tar-type arguments, we accept pxzf spew(SPEW_DEBUG, "at_first=%d argv[0][0]=%c", at_first_arg, argv[0][0]); const char *arg = argv[0]; int has_named_input = 0; while (arg[0]) { spew(SPEW_DEBUG, "arg=%s", arg); switch (arg[0]) { case 'p': arg_preserve = 1; break; case 'c': case 'x': if (tarmode != ' ') { fprintf(stderr, "incompatible: '%c' and '%c'\n", tarmode, arg[0]); spew(SPEW_ERROR, "incompatible: '%c' and '%c", tarmode, arg[0]); help(1); return ERROR; } tarmode = arg[0]; break; // we can only extract case 'z': arg_gunzip = 1; break; case 'f': has_named_input = 1; break; default: fprintf(stderr, "Unknown option %c\n", arg[0]); spew(SPEW_ERROR, "Unknown option %c", arg[0]); help(1); return ERROR; break; } arg++; } if (has_named_input) { argc--; argv++; if (argc == 0) { fprintf(stderr, "Need a FILE argument for f\n"); spew(SPEW_DEBUG, "Need a FILE argument for f"); help(1); return ERROR; } if (strcmp(argv[0], "-") != 0) { arg_named_input = argv[0]; } } } else { vector_of_strings_push(vs, argv[0]); } argc--; argv++; at_first_arg = 0; } if (tarmode == ' ') { fprintf(stderr, "You need to specify exactly one of x or c.\n"); spew(SPEW_ERROR, "You need to specify exactly one of x or c."); help(1); return 1; } if (tarmode != 'c' && vector_of_strings_size(vs) > 0) { fprintf(stderr, "Don't specify files except when specifying c.\n"); spew(SPEW_DEBUG, "Don't specify files except when specifying c."); help(1); return ERROR; } return CONTINUE; } #define NAMELEN 600 // After doing everything, directories may need their permission and // owner set. We need to do this from the bottom up in the directory // hierarchy, since if we change the permissions on the root of the // hierarchy, we may be unable to modify anything else. We'll do it // in waves, based on the number of slashes in the directory name. struct vecitem { // A directory to fix up. char *fullname; size_t mode; size_t uid; size_t gid; size_t mtime; }; // Protected by the fixup_mutex static size_t fixup_slash_count; // the size being worked on right now static size_t n_fixing_up = 0; // How many are working on fixup slash count. static struct vec *fixup_dirs; static pthread_mutex_t fixup_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t fixup_cond = PTHREAD_COND_INITIALIZER; static size_t countslash(const char *s) { size_t count = 0; while (*s) { if ((*s) == '/') count++; s++; } return count; } static int compare_number_of_slashes(const struct vecitem *a, const struct vecitem *b) { size_t as = countslash(a->fullname); size_t bs = countslash(b->fullname); if (as < bs) { return -1; } if (as > bs) { return +1; } return 0; } static void* fixup_runner(void *ignore __attribute__((__unused__))) { pthread_mutex_lock(&fixup_mutex); while (1) { while (1 && vec_size(fixup_dirs) && n_fixing_up && countslash(vec_peek(fixup_dirs)->fullname) < fixup_slash_count) { // If n_dirs_to_fixup becomes zero, then the // fixup_slash_count will also have changed (and a // broadcast happens when the fixup_slash_count // changes). // // If n_fixing_up becomes zero then we really only care if // the fixup_slash_count changes (in whichcase a broadcast // happened) or the n_dirs_to_fixup goes to 0 (in which // case we broadcast). // // If the fixup_slash_count changes we broadcast. pthread_cond_wait(&fixup_cond, &fixup_mutex); } if (vec_size(fixup_dirs) == 0) { break; // No more work to do } size_t sc = countslash(vec_peek(fixup_dirs)->fullname); if (sc == fixup_slash_count) { n_fixing_up++; struct vecitem *dtf = vec_pop(fixup_dirs); spew(SPEW_DEBUG, "working on %s", dtf->fullname); pthread_mutex_unlock(&fixup_mutex); if (arg_preserve && dtf->uid != SIZE_MAX) { int r __attribute__((unused)) = chown(dtf->fullname, (uid_t)dtf->uid, (gid_t)dtf->gid); // Don't complain if this goes wrong } { int r __attribute__((unused)) = chmod(dtf->fullname, (mode_t)dtf->mode); // Don't complain if this goes wrong } if (dtf->mtime) { struct timespec times[2] = {{.tv_nsec=UTIME_OMIT}, {.tv_sec = (time_t)dtf->mtime, .tv_nsec=0}}; int r = utimensat(AT_FDCWD, dtf->fullname, times, AT_SYMLINK_NOFOLLOW); assert(r == 0); } FREE(dtf->fullname); FREE(dtf); pthread_mutex_lock(&fixup_mutex); n_fixing_up--; if (n_fixing_up == 0 && vec_size(fixup_dirs) == 0) { pthread_cond_broadcast(&fixup_cond); } } else { assert(n_fixing_up == 0); fixup_slash_count = sc; { // debug code size_t count = 0; for (size_t i = 0; i < vec_size(fixup_dirs); i++) { if (countslash(vec_fetch(fixup_dirs, i)->fullname) == sc) count++; } spew(SPEW_DEBUG, "sc=%ld (count=%lu)", sc, count); } pthread_cond_broadcast(&fixup_cond); } } pthread_mutex_unlock(&fixup_mutex); return NULL; } static void Close(int fd) { int r = close(fd); assert(r == 0); } static void Dup2(int oldfd, int newfd) { int r = dup2(oldfd, newfd); if (r != newfd) { fprintf(stderr, "dup2(%d %d) -> %d --> errno=%d (%s)\n", oldfd, newfd, r, errno, strerror(errno)); spew(SPEW_DEBUG, "dup2(%d %d) -> %d --> errno=%d (%s)", oldfd, newfd, r, errno, strerror(errno)); } } enum { READ_END = 0, WRITE_END = 1 }; static int Pipefrom(int fd, const char *execarg, ...) { size_t n = 0; char **MALLOC_N(argv, 1); va_list ap; va_start(ap, execarg); while (1) { REALLOC(argv, n+1); char *s = va_arg(ap, char *); argv[n++] = s; if (s == NULL) break; } va_end(ap); int pipefd[2]; { int r = pipe(pipefd); assert(r == 0); } if (fork() == 0) { Dup2(fd, STDIN_FILENO); Close(fd); Dup2(pipefd[WRITE_END], STDOUT_FILENO); Close(pipefd[WRITE_END]); Close(pipefd[READ_END]); execvp(execarg, argv); assert(0); } else { Close(fd); Close(pipefd[WRITE_END]); FREE(argv); return pipefd[READ_END]; } } static int Pipeto(int fd, const char *execarg, ...) { size_t n = 0; char **MALLOC_N(argv, n); va_list ap; va_start(ap, execarg); while (1) { REALLOC(argv, n+1); char *s = va_arg(ap, char *); argv[n++] = s; if (s == NULL) break; } va_end(ap); int pipefd[2]; { int r = pipe(pipefd); assert(r == 0); } if (fork() == 0) { Dup2(fd, STDOUT_FILENO); Close(fd); Dup2(pipefd[READ_END], STDIN_FILENO); Close(pipefd[WRITE_END]); Close(pipefd[READ_END]); execvp(execarg, argv); assert(0); } else { Close(fd); Close(pipefd[READ_END]); return pipefd[WRITE_END]; } } struct stringset *directories_i_know_about; static void dirname_of_pathname(const char *pathname, char *dirname) { // find the destination directory name from the full pathname size_t filelen = strlen(pathname) + 1; strncpy(dirname, pathname, filelen); const size_t nl = strlen(dirname); if (nl != 0 && dirname[nl-1] == '/') { dirname[nl-1] = 0; } char *lastslash = strrchr(dirname, '/'); if (lastslash) { *(lastslash+1) = 0; } else { dirname[0] = 0; // no slashes, so use the empty dir } } static void ensure_directory_exists(const char *dirname, int implicitly_create) // implicitly_create is 1 for directories that are not explicitly created, // so we should simply mkdir using the default permissions implied by // whatever umask is. { if (dirname[0] && !stringset_contains(directories_i_know_about, dirname) != 0) { // Don't know about it. size_t dirlen = strlen(dirname)+1; char *parentdir = malloc(dirlen); memset(&parentdir[0], 0, dirlen); dirname_of_pathname(dirname, parentdir); ensure_directory_exists(parentdir, 1); if (implicitly_create) { int r = mkdir(dirname, 0777); if (r != 0 && errno != EEXIST) { fprintf(stderr, "Trying to mkdir %s\n", dirname); perror("mkdir"); fprintf(stderr, "I'll try to continue\n"); spew(SPEW_ERROR, "mkdir %s failed errno=%d (%s)", dirname, errno, strerror(errno)); } } else { struct fwobject *b = mk_fwobject('5', dirname, NULL, 0, 0, 0, 0, 0, NULL); fwg_add2(fwg, dirname, b, sizeof(*b) + strlen(dirname), dirname, parentdir); } stringset_insert(directories_i_know_about, dirname); if(parentdir) { FREE(parentdir); } } } static size_t extracttar(void) // Return the error count. { size_t errcount = 0; int input_fd = arg_named_input ? open(arg_named_input, O_RDONLY | O_CLOEXEC) : dup(STDIN_FILENO); if(input_fd < 0) { fprintf(stderr, "No such tar file exists : %s\n", arg_named_input); errcount++; return errcount; } int pv_fd = arg_show_progress ? Pipefrom(input_fd, "pv", "pv", NULL) : input_fd; assert(pv_fd >= 0); int unzipped_input = arg_gunzip ? Pipefrom(pv_fd, "gunzip", "gunzip", "-c", NULL) : pv_fd; assert(unzipped_input); FILE *input_file = fdopen(unzipped_input, "r"); if (arg_change_dir) { int r = chdir(arg_change_dir); if (r != 0) { fprintf(stderr, "Couldn't change to dir %s\n", arg_change_dir); perror("chdir"); spew(SPEW_ERROR, "chdir %s failed errno=%d (%s)", arg_change_dir, errno, strerror(errno)); exit(1); } } fixup_dirs = mk_vec(); directories_i_know_about = mk_stringset(); pinit(); start_runners(); int saw_end = 0; size_t count = 0; char prev_prev_typeflag = 0; char prev_typeflag = 0; // For long names the encoding is to say (with 'L') "next one or more // 512-byte block contains the name of the file for the following // record". Similarly to say with 'K' that the next one or more 512-byte // block contains the linkname for the file in the following // record". And if there is an L followed by a K then they both // modify the following record. // Here the long_name_from_prev is supposed to store the full name of // the file for encoding = 'L' which can span across multiple blocks. int has_name_from_prev = 0; char *long_name_from_prev = NULL; int has_linkname_from_prev = 0; struct posix_header linkname_from_prev; linkname_from_prev.name[0] = 0; size_t regcount = 0, linkcount = 0, symlinkcount = 0, dircount = 0; size_t n_records_seen = 0; while (1) { if (prev_typeflag != 'L' && prev_typeflag != 'K') { has_name_from_prev = 0; has_linkname_from_prev = 0; } char *fullname; char linkname[600]; struct posix_header h; { static struct posix_header zero; h = zero; size_t r = fread(&h, 1, sizeof(h), input_file); if (r == 0 && feof(input_file)) { break; } if (n_records_seen == 0 && memcmp(h.magic, "ustar", 5) != 0) { fprintf(stderr, "Input does not appear to be a tar file.\n"); spew(SPEW_ERROR, "Input does not appear to be a tar file."); if (0 == memcmp(h.name, (unsigned char[]){0x1f, 0x8b, 0x08}, 3)) { fprintf(stderr, "Input appears to be gzipped, "); spew(SPEW_ERROR, "Input appears to be gzipped, "); if (arg_gunzip) { fprintf(stderr, "which is confusing since you specified -z\n"); spew(SPEW_ERROR, "which is confusing since you specified -z"); } else { fprintf(stderr, "without specifying -z, e.g., as\n" " %s xzf file.tar.gz\n", cmd); spew(SPEW_ERROR, "without specifying -z, e.g., as\n" " %s xzf file.tar.gz", cmd); } } else if (0 == memcmp( h.name, (unsigned char[]){0xFD, '7', 'z', 'X', 'Z'}, 5)) { fprintf(stderr, "Input appears to be xz compressed," " which %s cannot (yet) decompress.\n", cmd); fprintf(stderr, "Maybe you can use an external decompressor," " e.g., as\n xzcat file.tar.xz|%s xf -\n", cmd); spew(SPEW_ERROR, "Input appears to be xz compressed," " which %s cannot (yet) decompress.", cmd); spew(SPEW_ERROR, "Maybe you can use an external decompressor," " e.g., as\n xzcat file.tar.xz|%s xf -", cmd); } else { fprintf(stderr, "Maybe it needs to be decompressed.\n"); spew(SPEW_DEBUG, "Maybe it needs to be decompressed."); } exit(1); } assert(r == sizeof(h)); n_records_seen++; } count++; { char *hname = h.name; size_t hname_limit = sizeof(h.name); if (has_name_from_prev) { hname = long_name_from_prev; spew(SPEW_DEBUG, "prev record name = %s", hname); hname_limit = strlen(long_name_from_prev); } else { if(long_name_from_prev) { FREE(long_name_from_prev); } } // The tar docs say that linkname is NUL terminated, but it's not. if (has_linkname_from_prev) { snprintf(linkname, sizeof(linkname), "%.*s", (int)512, linkname_from_prev.name); spew(SPEW_DEBUG, "p linkname=%s (%ld)", linkname, sizeof(linkname_from_prev.name)); } else { snprintf(linkname, sizeof(linkname), "%.*s", (int)sizeof(h.linkname), h.linkname); spew(SPEW_DEBUG, "h linkname=%s", linkname); } if (h.prefix[0]) { fullname = malloc(hname_limit + sizeof(h.prefix) + 1); int flen = snprintf(fullname, hname_limit + sizeof(h.prefix) + 1, "%.*s/%.*s", (int)sizeof(h.prefix), h.prefix, (int)hname_limit, hname); assert((size_t)flen <= (hname_limit + sizeof(h.prefix))); } else { fullname = malloc(hname_limit + 1); int flen = snprintf(fullname, hname_limit + 1, "%.*s", (int)hname_limit, hname); assert((size_t)flen <= hname_limit); } spew(SPEW_DEBUG, "prefix=%s fullname=%s", h.prefix, fullname); } int exclude_me = matches_excludes(fullname); spew(SPEW_DEBUG, "Exclude %s --> %d", fullname, exclude_me); size_t dirlen = strlen(fullname)+1; char *dirname = malloc(dirlen); memset(dirname, 0, strlen(fullname)); dirname_of_pathname(fullname, dirname); spew(SPEW_DEBUG, "dirname=%s sizeof(dirname)=%lu fullname=%s", dirname, sizeof(dirname), fullname); if (!exclude_me) ensure_directory_exists(dirname, 1); assert(h.pad[0]==0); size_t size_to_write = parse_octal_number(h.size, sizeof(h.size)); size_t size_to_read = ((size_to_write+511)/512)*512; if (size_to_write) { spew(SPEW_DEBUG, "Write size=%lu", size_to_write); } if (size_to_read) { spew(SPEW_DEBUG, "Read size=%lu ", size_to_read); } spew(SPEW_DEBUG, "type=%c", *h.typeflag); size_t mode = parse_octal_number(h.mode, sizeof(h.mode)); if (*h.typeflag == '5') { // Mkdir. if (!exclude_me) { dircount++; ensure_directory_exists(fullname, 0); struct vecitem *MALLOC(dtf); *dtf = (struct vecitem){ strdup(fullname), mode, parse_octal_number(h.uid, sizeof(h.uid)), parse_octal_number(h.gid, sizeof(h.gid)), parse_octal_number(h.mtime, sizeof(h.mtime)) }; vec_push(fixup_dirs, dtf); } } else if (*h.typeflag == '0') { regcount++; // Regular file. spew(SPEW_DEBUG, "insert regular file %s size=%lu mode=0%lo", fullname, size_to_write, mode); if (exclude_me) { size_t bufsize = 16ul * 1024ul * 1024ul; char *bytes = malloc(bufsize); while (size_to_read > 0) { size_t read_now = smin(size_to_read, bufsize); size_t s = do_read(input_file, bytes, read_now); if (s != read_now) { fprintf(stderr, "Tried to read %ld bytes, got %ld\n", read_now, s); spew(SPEW_WARN, "Tried to read %ld bytes, got %ld", read_now, s); } assert(s == read_now); size_to_read -= read_now; } free(bytes); } else if (size_to_read * 2 > memory_budget_MiB * MiB) { int done = 0; struct fwobject *b = mk_fwobject_read_from_input( *h.typeflag, fullname, size_to_write, mode, parse_octal_number(h.uid, sizeof(h.uid)), parse_octal_number(h.gid, sizeof(h.gid)), parse_octal_number(h.mtime, sizeof(h.mtime)), input_file, size_to_read, &done); fwg_add2(fwg, fullname, b, sizeof(*b) + strlen(fullname), fullname, dirname); pthread_mutex_lock(&fwmutex); while (!done) { pthread_cond_wait(&fwcond, &fwmutex); } pthread_mutex_unlock(&fwmutex); spew(SPEW_DEBUG, "main proceeding"); } else { // malloc and free are fine with size_to_read==0. char *bytes = malloc(size_to_read); spew(SPEW_DEBUG, "malloc(%zu) = %p", size_to_read, bytes); if (size_to_read) { size_t s = do_read(input_file, bytes, size_to_read); assert(s == size_to_read); } struct fwobject *b = mk_fwobject(*h.typeflag, fullname, NULL, size_to_write, mode, parse_octal_number(h.uid, sizeof(h.uid)), parse_octal_number(h.gid, sizeof(h.gid)), parse_octal_number(h.mtime, sizeof(h.mtime)), bytes); fwg_add2(fwg, fullname, b, sizeof(*b) + strlen(fullname) + size_to_write, fullname, dirname); } } else if (*h.typeflag == 0) { if (saw_end == 0) { saw_end = 1; } else { if (dirname) { FREE(dirname); } if (fullname) { FREE(fullname); } break; } } else if (*h.typeflag == 'g') { spew(SPEW_DEBUG, "global extended header"); char *bytes = malloc(size_to_read); assert(bytes); size_t s = do_read(input_file, bytes, size_to_read); assert(s == size_to_read); free(bytes); } else if (*h.typeflag == '1') { // We have to wait on the fullname and the linkname. It's // possible that the directories are the same. linkcount++; assert(linkname[0] && fullname[0]); // If the fullname and the linkname are the same, then // don't do anything. That might not be right if the new // link has different permissions or something. if (!exclude_me && strcmp(fullname, linkname) != 0) { struct fwobject *b = mk_fwobject(*h.typeflag, fullname, linkname, 0, 0, 0, 0, 0, NULL); spew(SPEW_DEBUG, "adding '%c'", *h.typeflag); fwg_add3(fwg, fullname, b, sizeof(*b) + strlen(fullname) + strlen(linkname), fullname, dirname, linkname); } } else if (!exclude_me && (*h.typeflag == '2' // symlink || *h.typeflag == '6' // fifo )) { if (*h.typeflag == 2) symlinkcount++; struct fwobject *b = mk_fwobject(*h.typeflag, fullname, *h.typeflag == '2' ? linkname : NULL, 0, mode, 0, 0, 0, NULL); spew(SPEW_DEBUG, "adding '%c'", *h.typeflag); fwg_add2(fwg, fullname, b, sizeof(*b) + strlen(fullname) + strlen(linkname), fullname, dirname); } else if (*h.typeflag == 'L') { if (0 && prev_typeflag != '5' && prev_typeflag != '0') { fprintf(stderr, "L: prev_typeflag = %c\n", prev_typeflag); spew(SPEW_DEBUG, "L: prev_typeflag = %c", prev_typeflag); } long_name_from_prev = malloc(size_to_read); size_t r = fread(long_name_from_prev, 1, size_to_read, input_file); assert(r == size_to_read); spew(SPEW_DEBUG, "Long name: name=%s\n", long_name_from_prev); has_name_from_prev = 1; } else if (*h.typeflag == 'K') { if (0 && prev_typeflag != '5' && prev_typeflag != '0') { fprintf(stderr, "K: prev_typeflag = %c\n", prev_typeflag); spew(SPEW_DEBUG, "K: prev_typeflag = %c", prev_typeflag); } size_t r = fread(&linkname_from_prev, 1, sizeof(linkname_from_prev), input_file); assert(r == sizeof(linkname_from_prev)); has_linkname_from_prev = 1; } else { fprintf(stderr, "h.typeflag=%c (prev_typeflag=%c) (prev_prev_typeflag=%c)\n", *h.typeflag, prev_typeflag, prev_prev_typeflag); spew(SPEW_DEBUG, "h.typeflag=%c (prev_typeflag=%c) (prev_prev_typeflag=%c)", *h.typeflag, prev_typeflag, prev_prev_typeflag); debug_print_posix_header(&h); abort(); } prev_prev_typeflag = prev_typeflag; prev_typeflag = *h.typeflag; if (dirname) { FREE(dirname); } if (fullname) { FREE(fullname); } } fwg_end_of_nodes(fwg); runners_finish(); if (vec_size(fixup_dirs)) { const size_t time_fixup = 0; struct timespec start_fixup, end_fixup; if (time_fixup) { clock_gettime(CLOCK_MONOTONIC, &start_fixup); fprintf(stderr, "Fixing up, there are %lu dirs\n", vec_size(fixup_dirs)); spew(SPEW_DEBUG, "Fixing up, there are %lu dirs", vec_size(fixup_dirs)); } vec_sort(fixup_dirs, compare_number_of_slashes); fixup_slash_count = countslash(vec_peek(fixup_dirs)->fullname); pthread_t *MALLOC_N(fthreads, n_threads); for (size_t i = 0; i < n_threads; i++) { pthread_create(&fthreads[i], NULL, fixup_runner, NULL); } for (size_t i = 0; i < n_threads; i++) { void *v; pthread_join(fthreads[i], &v); } FREE(fthreads); if (time_fixup) { clock_gettime(CLOCK_MONOTONIC, &end_fixup);; fprintf(stderr, "Fixup took %6.3fs\n", (double)(end_fixup.tv_sec - start_fixup.tv_sec) + 1e-9*(double)(end_fixup.tv_nsec - start_fixup.tv_nsec)); spew(SPEW_DEBUG, "Fixup took %6.3fs", (double)(end_fixup.tv_sec - start_fixup.tv_sec) + 1e-9*(double)(end_fixup.tv_nsec - start_fixup.tv_nsec)); } } // In some cases the dirs_to_fixup are still there (e.g., if we aren't preserving) while (1) { struct vecitem *item = vec_pop(fixup_dirs); if (item == NULL) { break; } FREE(item->fullname); FREE(item); } pdestroy(); fclose(input_file); directories_i_know_about = stringset_destroy(directories_i_know_about); fixup_dirs = vec_destroy(fixup_dirs); spew(SPEW_DEBUG, "regcount=%ld linkcount=%ld symlinkcount=%ld dircount=%lu", regcount, linkcount, symlinkcount, dircount); while (1) { int status; int r = wait(&status); if (r == -1 && errno == ECHILD) { break; } if (r == -1) { fprintf(stderr, "%s: waiting on child %s\n", cmd, strerror(errno)); spew(SPEW_DEBUG, "%s: waiting on child %s", cmd, strerror(errno)); errcount++; break; } if (!WIFEXITED(status)) { fprintf(stderr, "%s: Child did not return normally\n", cmd); spew(SPEW_ERROR, "%s: Child did not return normally", cmd); errcount++; } else if (WEXITSTATUS(status) != 0) { fprintf(stderr, "%s: Child returned status %d\n", cmd, WEXITSTATUS(status)); spew(SPEW_ERROR, "%s: Child returned status %d", cmd, WEXITSTATUS(status)); errcount++; } } if (errcount == 0 && n_records_seen == 0) { fprintf(stderr, "%s: This does not look like a tar archive\n", cmd); spew(SPEW_ERROR, "%s: This does not look like a tar archive", cmd); errcount++; } return errcount; } // For createtar static FILE *output_file; static size_t n_written = 0; static pthread_mutex_t emit_mutex = PTHREAD_MUTEX_INITIALIZER; static unsigned char ph_string[512]; static size_t ph_off; static void e_char(unsigned char c) { ph_string[ph_off++] = c; } static void e_string_short(const char *string, size_t len) { assert(ph_off + len <= sizeof(ph_string)); size_t slen = strlen(string); if(slen < len) { memcpy(ph_string + ph_off, string, slen); } else { memcpy(ph_string + ph_off, string, len); } while (slen < len) { ph_string[ph_off + slen++] = 0; } ph_off += len; } static void e_string(const char *string, size_t len) { assert(strlen(string) <= len); e_string_short(string, len); } static void e_number(size_t number, size_t len) { int r = snprintf((char*)ph_string+ph_off, 512-ph_off, "%.*zo", (int)len-1, number); assert(r >= 0); assert(ph_off + (size_t)r < 512); if ((size_t)r > len-1) { // It's too big, and must be represented in binary. assert((len > sizeof(size_t)) || (0 == (number >> (8 * len - 1)))); for (size_t i = len; i > 0; i--) { unsigned char extra_bits = (i == 1) ? 0x80u : 0; unsigned char the_byte = (unsigned char)(number & 0xffu); ph_string[ph_off + i - 1] = extra_bits | the_byte; number >>= 8; } } ph_off += len; } static void e_checksum_blanks(void) { for (size_t i = 0; i < 8; i++) e_char(' '); } static void e_prepare_checksum(void) { size_t checksum = 0; for (size_t i = 0; i < 512; i++) checksum += (unsigned char)ph_string[i]; snprintf((char*)ph_string+148, 8, "%06zo", checksum); } static void emit_longlink(const char *fname, unsigned char typ) { e_string("././@LongLink", 100); e_number(0644, 8); e_number(0, 8); e_number(0, 8); e_number(strlen(fname)+1, 12); e_number(0, 12); e_checksum_blanks(); e_char(typ); e_string("", 100); e_string("ustar ", 8); e_string("root", 32); e_string("root", 32); e_string("", 8+8+155+12); assert(ph_off == 512); e_prepare_checksum(); fwrite(ph_string, 1, ph_off, output_file); n_written += ph_off; ph_off = 0; fputs(fname, output_file); n_written += strlen(fname); size_t i = strlen(fname); if(!(i % 512)) { putc(0, output_file); n_written++; i++; } for (; i % 512; i++) { putc(0, output_file); n_written++; } } static void emit_header_locked(const char *fname, struct stat *statbuf, const char *link_target, unsigned char link_typeflag) { // To get it going, assume that the fname is less than 100 chars ph_off = 0; if (link_target) { if (strlen(link_target) > 100) { emit_longlink(link_target, 'K'); } } if (S_ISDIR(statbuf->st_mode)) { char *output_fname; int r = asprintf(&output_fname, "%s/", fname); assert(r >= 0); if (strlen(fname)+1 > 100) { emit_longlink(output_fname, 'L'); e_string_short(output_fname, 100); } else { e_string(output_fname, 100); } free(output_fname); } else { if (strlen(fname) > 100) { emit_longlink(fname, 'L'); e_string_short(fname, 100); } else { e_string(fname, 100); } } e_number(statbuf->st_mode & 07777, 8); e_number(statbuf->st_uid, 8); e_number(statbuf->st_gid, 8); assert(statbuf->st_size >= 0); if (S_ISLNK(statbuf->st_mode) || S_ISDIR(statbuf->st_mode) || link_target) { e_number(0, 12); } else { e_number((size_t)statbuf->st_size, 12); } e_number((size_t)statbuf->st_mtim.tv_sec, 12); e_checksum_blanks(); if (link_typeflag) { e_char(link_typeflag); } else if (S_ISREG(statbuf->st_mode)) { e_char('0'); } else if (S_ISLNK(statbuf->st_mode)) { e_char('2'); } else if (S_ISDIR(statbuf->st_mode)) { e_char('5'); } else if (S_ISFIFO(statbuf->st_mode)) { e_char('6'); } else { abort(); } if (S_ISLNK(statbuf->st_mode)) { assert(link_target); if (strlen(link_target) <= 100) { e_string(link_target, 100); } else { e_string_short(link_target, 100); } } else { if (link_target && strlen(link_target) <= 100) { e_string(link_target, 100); } else { e_string("", 100); } } e_string("ustar ", 8); { // uname struct passwd *pw = getpwuid(statbuf->st_uid); if (pw == NULL) { e_string("", 32); } else { e_string(pw->pw_name, 32); } } { // gname struct group *g = getgrgid(statbuf->st_gid); if (g == NULL) { e_string("", 32); } else { e_string(g->gr_name, 32); } } e_string("", 8); // devmajor e_string("", 8); // devminor e_string("", 155); e_string("", 12); assert(ph_off == 512); e_prepare_checksum(); fwrite(ph_string, 1, ph_off, output_file); ph_off = 0; n_written += 512; } static void emit_data_locked(const char *fname, size_t size) { FILE *f = fopen(fname, "r"); if (f == NULL) { return; } const size_t size_limit = 1024*1024u; // bigger than this, we grab the lock early char *buf=malloc(size_limit); size_t remaining_size = size; while (remaining_size) { size_t read_now = remaining_size > size_limit ? size_limit : remaining_size; size_t actual_read = fread(buf, 1, read_now, f); size_t actual_write = fwrite(buf, 1, actual_read, output_file); n_written += actual_read; remaining_size -= actual_write; } while (n_written % 512) { n_written++; putc(' ', output_file); } fclose(f); FREE(buf); } struct inodemap *inset; static int maybe_emit_link_locked(const char *fname, struct stat *statbuf) { void const *value; if (inodemap_lookup(inset, statbuf->st_ino, statbuf->st_dev, &value)) { emit_header_locked(fname, statbuf, value, '1'); return 1; } else { inodemap_insert(inset,statbuf->st_ino, statbuf->st_dev, strdup(fname)); return 0; } } static void emit(const char *fname, struct stat *statbuf) { const off_t size_limit = 1024*1024u; // bigger than this, we grab the lock early if (S_ISREG(statbuf->st_mode)) { if (statbuf->st_size > size_limit) { pthread_mutex_lock(&emit_mutex); if (!maybe_emit_link_locked(fname, statbuf)) { emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0); emit_data_locked(fname, (size_t)statbuf->st_size); } pthread_mutex_unlock(&emit_mutex); } else { assert(statbuf->st_size >= 0); char *buf = malloc((size_t)statbuf->st_size); FILE *f = fopen(fname, "r"); if (f == NULL) { FREE(buf); return; } size_t s = statbuf->st_size == 0 ? 0 : do_read(f, buf, (size_t)statbuf->st_size); assert(s == (size_t)statbuf->st_size); fclose(f); pthread_mutex_lock(&emit_mutex); if (!maybe_emit_link_locked(fname, statbuf)) { emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0); fwrite(buf, 1, s, output_file); size_t target = ((s+511)/512)*512; for (size_t i = s; i < target; i++) { putc(0, output_file); } n_written += target; } pthread_mutex_unlock(&emit_mutex); FREE(buf); } } else if (S_ISLNK(statbuf->st_mode)) { assert(statbuf->st_size >= 0); char *buf = malloc((size_t)statbuf->st_size + 2); ssize_t r = readlink(fname, buf, (size_t)statbuf->st_size); assert(r == statbuf->st_size); buf[statbuf->st_size] = 0; pthread_mutex_lock(&emit_mutex); if (!maybe_emit_link_locked(fname, statbuf)) { emit_header_locked(fname, statbuf, buf, '2'); } pthread_mutex_unlock(&emit_mutex); FREE(buf); } else { pthread_mutex_lock(&emit_mutex); if (!maybe_emit_link_locked(fname, statbuf)) { emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0); } pthread_mutex_unlock(&emit_mutex); } } static magic_t ctar_magic = "ctar magic"; struct ctarframe { magic_t *magic; struct vector_of_strings *vs; size_t inlet_sum; size_t sum; }; static struct ctarframe *mk_ctarframe(struct vector_of_strings *vs) { struct ctarframe *MALLOC(cf); *cf = (struct ctarframe){.magic = &ctar_magic, .vs = vs, .inlet_sum = 0, .sum = 0}; return cf; } static struct ctarframe *ctarframe_free(struct ctarframe *cf) { vector_of_strings_destroy(cf->vs); FREE(cf); return NULL; } static struct ctarframe *ctarframe_cast(void *app_frame) { struct ctarframe *cf = app_frame; assert(cf->magic == &ctar_magic); return cf; } static void ctarfun_return_inlet(void *parent_app_frame, size_t result) { struct ctarframe *cf = ctarframe_cast(parent_app_frame); cf->inlet_sum += result; } static size_t ctar_error_count = 0; static pthread_mutex_t ctar_error_count_mutex = PTHREAD_MUTEX_INITIALIZER; static void bump_ctar_error_count(void) { pthread_mutex_lock(&ctar_error_count_mutex); ctar_error_count++; pthread_mutex_unlock(&ctar_error_count_mutex); } static struct frame *ctarfun2(struct frame *frame, void *app_frame); static struct frame*ctarfun(struct frame *frame, void *app_frame) { struct ctarframe *cf = ctarframe_cast(app_frame); char *fname; while ((fname = vector_of_strings_pop(cf->vs))) { if (!matches_excludes(fname)) { struct stat statbuf; { int r = lstat(fname, &statbuf); if (r != 0) { fprintf(stderr, "%s: %s: Cannot stat: %s\n", cmd, fname, strerror(errno)); spew(SPEW_ERROR, "%s: %s: Cannot stat: %s", cmd, fname, strerror(errno)); bump_ctar_error_count(); continue; } } cf->sum++; emit(fname, &statbuf); if (S_ISDIR(statbuf.st_mode)) { struct vector_of_strings *vs = mk_vector_of_strings(); DIR *dir = opendir(fname); assert(dir); struct dirent *entry; while ((entry = readdir(dir))) { if (strcmp(entry->d_name, ".") == 0) continue; if (strcmp(entry->d_name, "..") == 0) continue; char *path = pathcat(fname, entry->d_name); vector_of_strings_push(vs, path); FREE(path); } free(fname); closedir(dir); struct ctarframe *subframe = mk_ctarframe(vs); return ctarfun( sched_spawn(frame, ctarfun, subframe, ctarfun_return_inlet), subframe); } } free(fname); } return sched_sync(frame, ctarfun2); } static struct frame *ctarfun2(struct frame *frame, void *app_frame) { struct ctarframe *cf = ctarframe_cast(app_frame); size_t sum = cf->inlet_sum + cf->sum; cf = ctarframe_free(cf); return sched_return(frame, sum); } static void do_createtar(struct vector_of_strings *vs) { struct ctarframe *cf = mk_ctarframe(vs); size_t n = prun(n_threads, ctarfun, cf); spew(SPEW_INFO, "Tarred %zu objects", n); } static size_t createtar(struct vector_of_strings *vs) { ctar_error_count = 0; int output_fd = arg_named_input ? open(arg_named_input, O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, 0666) : dup(STDOUT_FILENO); assert(output_fd >= 0); int unzipped_output = arg_gunzip ? Pipeto(output_fd, "gzip", "gzip", "-c", NULL) : output_fd; output_file = fdopen(unzipped_output, "w"); if (arg_change_dir) { int r = chdir(arg_change_dir); if (r != 0) { fprintf(stderr, "Couldn't change to dir %s\n", arg_change_dir); perror("chdir"); spew(SPEW_ERROR, "Couldn't change to dir %s errno=%d (%s)", arg_change_dir, errno, strerror(errno)) ; exit(1); } } do_createtar(vs); spew(SPEW_INFO, "n-written=%zu\n", n_written); while (n_written % (20*512u)) { // Tar puts at least 2 512-byte blocks at the end, and enough to add up to a multiple of 20 512 blocks. putc(0, output_file); n_written++; } fclose(output_file); return ctar_error_count++; } static void destroy_string(void *sv) { FREE(sv); } int main (int argc, char *argv[]) { int ret = 0; find_open_fds_at_start(); we_are_root = geteuid () == ROOT_UID; if (we_are_root) arg_preserve = 1; struct vector_of_strings *vs = mk_vector_of_strings(); excludes = mk_vector_of_strings(); #ifdef SPEW_ENABLED size_t spew_level = SPEW_DEBUG; spew_config(argc, argv, "partar_log", /* log roll time in minutes */1440, /* spew_to_stderr */0, /*debug=*/0); spew_set_level(spew_level); spew(SPEW_INFO, "partar %s, SPLASH!!!", gitrev); #endif ret = parse_args(argc, argv, vs); switch (ret) { case DONE: case ERROR: goto do_exit; break; case CONTINUE: ret = 0; break; default: ret = 1; goto do_exit; break; } // parsing done and there is work to do! spew(SPEW_DEBUG, "arg_change_dir = %s", arg_change_dir); spew(SPEW_DEBUG, "\targ_named_input = %s", arg_named_input); size_t errcount = 0; if (tarmode == 'x') { errcount = extracttar(); vector_of_strings_destroy(vs); } else if (tarmode == 'c') { inset = mk_inodemap(); errcount = createtar(vs); inset = inodemap_destroy(inset, destroy_string); } else { assert(0); } if (errcount || some_errors) { fprintf(stderr, "%s (%s %s): Exiting with failure status due to previous errors\n", cmd, VERSION, gitrev); spew(SPEW_ERROR, "%s (%s %s): Exiting with failure status due to previous errors", cmd, VERSION, gitrev); ret = 2; } do_exit: spew_teardown(); fdleak_check(); vector_of_strings_destroy(excludes); return ret; }