Project

General

Profile

Download (67.2 KB) Statistics
| Branch: | Tag: | Revision:

root/software/fss-parallel-tools/partar.c @ b1d8e8c9

7e8045d8 David Sorber
/**
* Copyright © 2019, Oracle and/or its affiliates. All rights reserved.
*
* Licensed under the Universal Permissive License v 1.0 as shown at
* http://oss.oracle.com/licenses/upl
*
*/
#define _GNU_SOURCE
#include "fdleak.h"
#include "vec.h"
#include "putils.h"
#include "fwg.h"
#include "inodemap.h"
#include "malloc.h"
#include "sched.h"
#include "stringset.h"
#include "vector-of-strings.h"
#include "version.h"
#include <assert.h>
#include <ctype.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <fnmatch.h>
#include <grp.h>
#include <pthread.h>
#include <pwd.h>
#include <stdarg.h>
#include <stdio.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <unistd.h>
#include "common/spew.h"
#include <ftw.h>


// Parallel tar. For parallel untar The main thread scans through the
// tar file. If it finds a directory, it creates it. If it finds a
// regular file, it mallocs a place to hold the contents of the file,
// scans everything into the alloced object, and stores the alloced
// object into a vector of items to work on. (The main thread also
// handles symlinks: rationale symlinks seem to be uncommon, and it's
// simpler this way.)
//
// Meanwhile, a bunch of threads grab items out of the vector. Each
// item involves creating a file and writing the bytes into the file.
// They grab a random item to reduce the chances that they are trying
// to create files in the same directory.
//
// We cannot handle the case where the same file appears twice in the
// tar file (presumably the semantics is that the second file "wins".)
// We use a hash table to detect that this happens. The hash table
// doesn't need a mutex since it's only accessed by the main thread.

/* tar Header Block, from POSIX 1003.1-1990. */

/* POSIX header. */

struct posix_header
{ /* byte offset */
char name[100]; /* 0 */
char mode[8]; /* 100 */
char uid[8]; /* 108 */
char gid[8]; /* 116 */
char size[12]; /* 124 */
char mtime[12]; /* 136 */
char chksum[8]; /* 148 */
char typeflag[1]; /* 156 */
char linkname[100]; /* 157 */
char magic[6]; /* 257 */
char version[2]; /* 263 */
char uname[32]; /* 265 */
char gname[32]; /* 297 */
char devmajor[8]; /* 329 */
char devminor[8]; /* 337 */
char prefix[155]; /* 345 */
/* 500 */
char pad[12];
};

static void print_byte(char c) {
if (isprint(c) && c != '\\') {
fprintf(stderr, "%c", c);
} else {
fprintf(stderr, "\\%03o", c);
}
}

static void print_bytes(const char *s, size_t len) {
fprintf(stderr, "\"");
for (size_t i = 0; i < len; i++) {
print_byte(s[i]);
}
fprintf(stderr, "\"");
}

#define PRINT_BYTES(name) (fprintf(stderr, "\n %8s = ", #name), print_bytes(h->name, sizeof(h->name)))

static void debug_print_posix_header(const struct posix_header *h) {
fprintf(stderr, "posix header {");
PRINT_BYTES(name);
PRINT_BYTES(mode);
PRINT_BYTES(uid);
PRINT_BYTES(gid);
PRINT_BYTES(size);
PRINT_BYTES(mtime);
PRINT_BYTES(chksum);
PRINT_BYTES(typeflag);
PRINT_BYTES(linkname);
PRINT_BYTES(magic);
PRINT_BYTES(version);
PRINT_BYTES(uname);
PRINT_BYTES(gname);
PRINT_BYTES(devmajor);
PRINT_BYTES(devminor);
PRINT_BYTES(prefix);
PRINT_BYTES(pad);
fprintf(stderr, "\n ... }\n");
}

enum { default_memory_budget_MiB = 100 };
enum { MiB = 1024*1024 };
static size_t memory_budget_MiB = default_memory_budget_MiB;

enum { max_name_length = 255 };

enum {
DONE = 0,
ERROR = 1,
CONTINUE = 2,
};

#define ROOT_UID 0
static int we_are_root;

static int arg_preserve = 0; // default is 1 if we are root
static int arg_gunzip = 0;
static int arg_show_progress = 0;
static const char *arg_named_input = NULL;
static const char *arg_change_dir = NULL;

pthread_cond_t fwcond = PTHREAD_COND_INITIALIZER;
pthread_mutex_t fwmutex = PTHREAD_MUTEX_INITIALIZER;

struct fwobject {
char command;
char *fullname;
char *linkname;
size_t size;
size_t mode;
size_t uid;
size_t gid;
size_t mtime;
// If bytes!=NULL then the data is in bytes.
char *bytes;
//
// Else it's in input_file, to be gotten by do_read(input_file,
// size size_to_read), set *done and signal fwcond when done.
FILE *input_file;
size_t size_to_read;
int *done;
};
static struct fwobject *mk_fwobject(char command,
const char *fullname,
const char *linkname,
size_t size,
size_t mode,
size_t uid,
size_t gid,
size_t mtime,
char *bytes/*[size]*/) {
struct fwobject *MALLOC(b);
*b = (struct fwobject){.command = command,
.fullname = fullname ? strdup(fullname) : NULL,
.linkname = linkname ? strdup(linkname) : NULL,
.size = size,
.mode = mode,
.uid = uid,
.gid = gid,
.mtime = mtime,
.bytes = bytes,
.input_file = NULL,
.size_to_read = 0,
.done = NULL};
return b;
}

static struct fwobject *mk_fwobject_read_from_input(char command,
const char *fullname,
size_t size,
size_t mode,
size_t uid,
size_t gid,
size_t mtime,
FILE *input_file,
size_t size_to_read,
int *done) {
struct fwobject *MALLOC(b);
*b = (struct fwobject){.command = command,
.fullname = fullname ? strdup(fullname) : NULL,
.linkname = NULL,
.size = size,
.mode = mode,
.uid = uid,
.gid = gid,
.mtime = mtime,
.bytes = NULL,
.input_file = input_file,
.size_to_read = size_to_read,
.done = done};
return b;
}


static magic_t mep_magic = "mep magic";

struct mep_extra {
magic_t *magic;
const char *fname;
};

static int matches_excludes_predicate(const char *pattern,
void *mepv) {
struct mep_extra *mep = mepv;
assert(mep->magic == &mep_magic);
spew(SPEW_DEBUG, "matching %s to %s", mep->fname, pattern);

const char *str = mep->fname;
while (*str) {
spew(SPEW_DEBUG, "str=%s", str);
const char *end = index(str, '/');
if (end == NULL) {
spew(SPEW_DEBUG, "matching %s to %s", str, pattern);
int r = fnmatch(pattern, str, 0);
spew(SPEW_DEBUG, "%d", !r);
return !r;
} else if (strcmp(end, str) == 0) {
str = end+1;
spew(SPEW_DEBUG, "str=%s end=%s", str,end);
} else {
char path[256];
assert(end > str);
assert((size_t)(end-str) < sizeof(path) - 1);
memcpy(path, str, (size_t)(end-str));
path[end-str] = 0;
spew(SPEW_DEBUG, "matching %s to %s", path, pattern);
if (fnmatch(pattern, path, 0) == 0) {
spew(SPEW_DEBUG, "%s matches %s (original %s)", path, pattern, mep->fname);
return 1;
}
str = end+1;
spew(SPEW_DEBUG, "now str=%s", str);
}
}
spew(SPEW_DEBUG, "nope");
return 0;
}

int some_errors = 0;

static struct vector_of_strings *excludes;
static int matches_excludes(const char *fname) {
struct mep_extra mep = {&mep_magic, fname};
return vector_of_strings_any(excludes,
matches_excludes_predicate,
&mep);
}

static struct fwg *fwg;

static void pinit(void) {
fwg = mk_fwg(memory_budget_MiB * 1024ul * 1024ul);
};
static void pdestroy(void) {
fwg = fwg_destroy(fwg);
}

static size_t smin(size_t a, size_t b) {
if (a < b) return a;
else return b;
}

/*
* The function checks the stat of the files that already exist.
* It then compares the type of the existing file with the new file
* that needs to be created at the same path. In case of mismatch
* the existing file is deleted. If the existing file type is
* directory, it deletes the subtree as well.
*/
static int check_and_delete_conflicts(char typeflag, char *fullname) {
struct stat filestat;
char tempname[600];
snprintf(tempname, sizeof(tempname), "%s",
fullname);
if (tempname[strlen(tempname) - 1] == '/') {
tempname[strlen(tempname) - 1] = 0;
}
int ret = stat(tempname, &filestat);
if (ret == 0) {
if (S_ISDIR(filestat.st_mode) && typeflag != '5') {
/*
* Found a directory at the path where a new type
* of file is expected. Deleting along with subtree.
*/
delete_dir_recursively(tempname);
} else if ((S_ISREG(filestat.st_mode) && typeflag != '0')
|| (S_ISLNK(filestat.st_mode) && typeflag != '2')
|| (S_ISFIFO(filestat.st_mode) && typeflag != '6'))
{
do_unlink(tempname);
}
}
return 0;
}

static void process_fwobject(size_t threadn, struct fwobject *b, struct fwgnode *handle, int retry) {
int ret;
spew(SPEW_DEBUG, "cmd='%c' %s handle=%p", b->command, b->fullname, handle);
switch (b->command) {
case '0': {
// create file
spew(SPEW_DEBUG, "%lu: parallel (small) regular file \"%s\"", threadn, b->fullname);
chmod(b->fullname, 0700);
FILE *of = do_creat(b->fullname);
if (of == NULL && retry == 1) {
check_and_delete_conflicts(b->command, b->fullname);
process_fwobject(threadn, b, handle, 0);
return;
}
if (b->bytes) {
assert(!b->input_file && !b->size_to_read && !b->done);
do_write(of, b->bytes, b->size);
} else {
spew(SPEW_DEBUG, "Immediate read of %lu bytes into %lu bytes", b->size_to_read, b->size);
assert(b->input_file && b->size_to_read && b->done);
assert(!*b->done);
size_t bufsize = 16ul * 1024ul * 1024ul;
char *buf = malloc(bufsize);
size_t remaining_to_read = b->size_to_read;
size_t remaining_to_write = b->size;
while (remaining_to_read) {
assert(remaining_to_write <= remaining_to_read);
size_t read_now = smin(remaining_to_read, bufsize);
size_t r = fread(buf, 1, read_now, b->input_file);
assert(r <= read_now);
remaining_to_read -= read_now;
if (r < read_now) {
if (feof(b->input_file)) {
fprintf(stderr, "read from %s encountered premature end of file", b->fullname);
spew(SPEW_ERROR, "read from %s encountered premature end of file", b->fullname);
} else {
assert(ferror(b->input_file));
fprintf(stderr, "read from %s failed with error: %d\n", b->fullname, ferror(b->input_file));
spew(SPEW_ERROR, "read from %s failed with error: %d\n", b->fullname, ferror(b->input_file));
}
exit(1);
} else {
size_t write_now = smin(remaining_to_write, read_now);
size_t r2 = fwrite(buf, 1, write_now, of);
assert(r2 == write_now);
remaining_to_write -= write_now;
}
}
FREE(buf);
}
{
int r __attribute__((unused)) =
fchmod(fileno(of), (mode_t)b->mode);
// Don't check r. Either it worked or it didn't.
}
if (arg_preserve) {
int r __attribute__((unused)) =
fchown(fileno(of), (uid_t)b->uid, (gid_t)b->gid);
// Don't check r. Either it worked or it didn't. Gnu
// tar will try doing fchownat as a backup. Maybe
// we'll have to do that too...
}
do_fclose(of);
{
struct timespec times[2] = {{.tv_nsec=UTIME_OMIT},
{.tv_sec = (time_t)b->mtime, .tv_nsec=0}};
int r = utimensat(AT_FDCWD, b->fullname, times, AT_SYMLINK_NOFOLLOW);
if (r != 0) {
fprintf(stderr, "utimensat failed on %s errno=%d (%s)\n",
b->fullname, errno, strerror(errno));
spew(SPEW_ERROR, "utimensat failed on %s errno=%d (%s)",
b->fullname, errno, strerror(errno));
__sync_add_and_fetch(&some_errors, 1);
}
}
if (!b->bytes) {
*b->done = 1;
spew(SPEW_DEBUG, "Finished immediate read");
pthread_mutex_lock(&fwmutex);
pthread_cond_signal(&fwcond);
pthread_mutex_unlock(&fwmutex);
}
spew(SPEW_DEBUG, "finished %s", b->fullname);
FREE(b->fullname);
assert(!b->linkname);
FREE(b->bytes);
FREE(b);
break;
}
case '1': {
assert(b->fullname && b->fullname[0]);
assert(b->linkname && b->linkname[0]);
ret = do_link(b->linkname, b->fullname);
if (ret != 0 && retry == 1) {
check_and_delete_conflicts(b->command, b->fullname);
process_fwobject(threadn, b, handle, 0);
return;
}
FREE(b->linkname);
FREE(b->fullname);
FREE(b);
break;
}
case '2': {
ret = do_symlink(b->linkname, b->fullname);
if (ret != 0 && errno == EEXIST && retry == 1) {
check_and_delete_conflicts(b->command, b->fullname);
process_fwobject(threadn, b, handle, 0);
return;
}
{
struct timespec times[2] = {{.tv_nsec=UTIME_OMIT},
{.tv_sec = (time_t)b->mtime, .tv_nsec=0}};
int r = utimensat(AT_FDCWD, b->fullname, times, AT_SYMLINK_NOFOLLOW);
assert(r == 0);
}
FREE(b->linkname);
FREE(b->fullname);
FREE(b);
break;
}
case '5': {
ret = do_mkdir(b->fullname);
if (ret != 0 && retry == 1) {
check_and_delete_conflicts(b->command, b->fullname);
process_fwobject(threadn, b, handle, 0);
return;
}
FREE(b->fullname);
assert(!b->linkname);
FREE(b);
break;
}
case '6': {
ret = do_mkfifo(b->fullname, (mode_t)b->mode);
if (ret != 0 && retry == 1) {
check_and_delete_conflicts(b->command, b->fullname);
process_fwobject(threadn, b, handle, 0);
return;
}
FREE(b->fullname);
assert(!b->linkname);
FREE(b);
break;
}
default: {
fprintf(stderr, "Unknown command %c\n", b->command);
spew(SPEW_DEBUG, "Unknown command %c", b->command);
assert(0);
}
}
spew(SPEW_DEBUG, "thread %lu finishing an fwobject", threadn);
fwg_finish_node(fwg, handle);
spew(SPEW_DEBUG, "%lu: next", threadn);
}

static void process_fwobjects(size_t threadn) {
while (1) {
struct fwobject *b;
struct fwgnode *handle;

int r = fwg_get_ready_node(fwg, &b, &handle);
if (r != 0) {
return;
}
assert(b);
process_fwobject(threadn, b, handle, 1);
}
}

static void* fwobjects_runner(void *arg) {
size_t *ip = arg;
process_fwobjects(*ip);
return NULL;
}

enum { default_n_threads = 32 };
static size_t n_threads = default_n_threads;
static pthread_t *threads;
static size_t *threadid;

static void start_runners(void) {
MALLOC_N(threads, n_threads);
MALLOC_N(threadid, n_threads);
for (size_t i = 0; i < n_threads; i++) {
threadid[i]=i;
pthread_create(&threads[i], NULL, fwobjects_runner, &threadid[i]);
}
}
static void runners_finish(void) {
// After the last enqueue, call sfq_finish_up to wait for all the work to finish.
for (size_t i = 0; i < n_threads; i++) {
void *v;
pthread_join(threads[i], &v);
assert(v == NULL);
}
FREE(threads);
FREE(threadid);
}

static size_t parse_octal_number(char *field, size_t sizeof_field) {
assert(sizeof_field > 0);
if (*field == '\200') {
size_t result = 0;
for (size_t i = 1; i < sizeof_field; i++) {
result = (result * 256) + ((unsigned char *)field)[i];
}
return result;
} else {
assert(0 == (*(unsigned char*)field & 0x80));
errno = 0;
char *end;
size_t result = strtoul(field, &end, 8);
assert(errno == 0);
assert(*end == 0 || *end==' ');
assert(end < field+sizeof_field);
return result;
}
}

static const char *cmd;
static char tarmode = ' ';

void help(int exitcode) {
spew(SPEW_DEBUG, "exitcode %d ", exitcode);
FILE *out = exitcode ? stderr : stdout;
fprintf(out, "Usage: %s [TAROPTION] [OPTION] FILE...\n", cmd);
fprintf(out, "%s is a parallel implementation of tar. It can create and extract\n", cmd);
fprintf(out, "tarballs in parallel, which can sometimes provide improved performance.\n");
fprintf(out, " TAROPTION is only the single-letter tar options with no prefix.\n");
fprintf(out, " TAROPTION can be\n");
fprintf(out, " p Preserve permissions (default for superuser).\n");
fprintf(out, " c Create a tarball. Specify FILE(s).\n");
fprintf(out, " x Extract files. Don't specify FILE(s). You need exactly one of x or c.\n");
fprintf(out, " z Filter the archive through gzip(1).\n");
fprintf(out, " f The next argument names the archive (the input tarball for x, the output for c). A dash `-' means stdin/stdout. Without f, the default is stdin/stdout.\n");
fprintf(out, " OPTION can be\n");
fprintf(out, " -C DIR Change to directory DIR\n");
fprintf(out, " -h, --help Print help.\n");
fprintf(out, " -P P (capital P) use P-fold parallelism (default %d threads).\n", default_n_threads);
fprintf(out, " -m M Set memory budget to M mebibytes (default %d MiB).\n", default_memory_budget_MiB);
fprintf(out, " --exclude=PATTERN Exclude patterns matching PATTERN, a glob(3)-style wildcard pattern.\n");
fprintf(out, " -p, --preserve-permissions, --same-permissions\n");
fprintf(out, " extract information about file permissions (default for superuser)\n");
fprintf(out, " --version Print program version.\n");
fprintf(out, " --history Print a brief history of the versions.\n");
fprintf(out, " --progress Monitor progress of untarring.\n");
fprintf(out, "This is version %s gitrev %s compiled %s %s.\n",
VERSION, gitrev, __DATE__, __TIME__);
fprintf(out, "Report bugs to parallel-tools-support_ww_grp@oracle.com\n");
}

static int prefix_matches(const char *prefix, const char *string, char const **suffix) {
while (*prefix == *string) {
spew(SPEW_DEBUG, "prefix %s string %s", prefix, string);
prefix++; string++;
}
if (*prefix == 0) {
*suffix = string;
spew(SPEW_DEBUG, "prefix %s suffix %s string %s", prefix, *suffix, string);
return 1;
} else {
return 0;
}
}

static int parse_args(int argc, char *argv[],
struct vector_of_strings *vs) {
assert(argc > 0);
cmd = argv[0];
argc--; argv++;
int at_first_arg = 1;
char const *suffix;
while (argc) {
spew(SPEW_DEBUG, "at_first=%d arg parse = %s", at_first_arg, argv[0]);
if (strcmp(argv[0], "-P") == 0) {
argc--; argv++;
if (argc == 0) {
help(1);
return ERROR;
}
n_threads = parse_number_or_help(argv[0]);
if (n_threads == 0) {
// something is wrong should've got a value
return ERROR;
}
spew(SPEW_DEBUG, "number of threads = %ld", n_threads);
} else if (strcmp(argv[0], "-m") == 0) {
argc--; argv++;
if (argc == 0) {
help(1);
return ERROR;
}
memory_budget_MiB = parse_number_or_help(argv[0]);
if (memory_budget_MiB == 0) {
// something is wrong should've got a value
return ERROR;
}
spew(SPEW_DEBUG, "memory budget (MiB) = %ld", memory_budget_MiB);
} else if (strcmp(argv[0], "-h") == 0 ||
strcmp(argv[0], "--help") == 0) {
help(0);
return DONE;

} else if (strcmp(argv[0], "--version") == 0) {
printf("%s %s\n"
"Copyright (C) 2019 Oracle.\n"
"Written by Bradley C. Kuszmaul\n",
cmd, VERSION);
return DONE;
} else if (strcmp(argv[0], "--history") == 0) {
fprintf(stderr, "%s", VERSION_HISTORY);
return DONE;
} else if (strcmp(argv[0], "-C") == 0) {
argc--; argv++;
if (argc == 0) {
fprintf(stderr, "Need a DIR argument for -C\n");
spew(SPEW_ERROR, "Need a DIR argument for -C");
help(1);
return ERROR;
}
arg_change_dir = argv[0];
} else if (strcmp(argv[0], "--progress") == 0) {
arg_show_progress = 1;
} else if (strcmp(argv[0], "--exclude") == 0) {
argc--; argv++;
if (argc == 0) {
fprintf(stderr, "Need a PATTERN argument for --exclude\n");
spew(SPEW_ERROR, "Need a PATTERN argument for --exclude");
help(1);
return ERROR;
}
spew(SPEW_DEBUG, "excluding %s", argv[0]);
vector_of_strings_push(excludes, argv[0]);
} else if (strcmp(argv[0], "-p") == 0
|| strcmp(argv[0], "--preserve-permissions") == 0
|| strcmp(argv[0], "--same-permissions") == 0) {
arg_preserve = 1;
} else if (prefix_matches("--exclude=", argv[0], &suffix)) {
spew(SPEW_DEBUG, "excluding %s suffix %s", argv[0], suffix);
vector_of_strings_push(excludes, suffix);
} else if (at_first_arg && argv[0][0] != '-') {
// tar-type arguments, we accept pxzf
spew(SPEW_DEBUG, "at_first=%d argv[0][0]=%c", at_first_arg, argv[0][0]);
const char *arg = argv[0];
int has_named_input = 0;
while (arg[0]) {
spew(SPEW_DEBUG, "arg=%s", arg);
switch (arg[0]) {
case 'p':
arg_preserve = 1;
break;
case 'c':
case 'x':
if (tarmode != ' ') {
fprintf(stderr, "incompatible: '%c' and '%c'\n",
tarmode, arg[0]);
spew(SPEW_ERROR, "incompatible: '%c' and '%c",
tarmode, arg[0]);
help(1);
return ERROR;
}
tarmode = arg[0];
break; // we can only extract
case 'z':
arg_gunzip = 1;
break;
case 'f':
has_named_input = 1;
break;
default:
fprintf(stderr, "Unknown option %c\n", arg[0]);
spew(SPEW_ERROR, "Unknown option %c", arg[0]);
help(1);
return ERROR;
break;
}
arg++;
}
if (has_named_input) {
argc--; argv++;
if (argc == 0) {
fprintf(stderr, "Need a FILE argument for f\n");
spew(SPEW_DEBUG, "Need a FILE argument for f");
help(1);
return ERROR;
}
if (strcmp(argv[0], "-") != 0) {
arg_named_input = argv[0];
}
}
} else {
vector_of_strings_push(vs, argv[0]);
}
argc--; argv++;
at_first_arg = 0;
}
if (tarmode == ' ') {
fprintf(stderr, "You need to specify exactly one of x or c.\n");
spew(SPEW_ERROR, "You need to specify exactly one of x or c.");
help(1);
return 1;
}
if (tarmode != 'c' && vector_of_strings_size(vs) > 0) {
fprintf(stderr, "Don't specify files except when specifying c.\n");
spew(SPEW_DEBUG, "Don't specify files except when specifying c.");
help(1);
return ERROR;
}
return CONTINUE;
}

#define NAMELEN 600

// After doing everything, directories may need their permission and
// owner set. We need to do this from the bottom up in the directory
// hierarchy, since if we change the permissions on the root of the
// hierarchy, we may be unable to modify anything else. We'll do it
// in waves, based on the number of slashes in the directory name.

struct vecitem { // A directory to fix up.
char *fullname;
size_t mode;
size_t uid;
size_t gid;
size_t mtime;
};

// Protected by the fixup_mutex
static size_t fixup_slash_count; // the size being worked on right now
static size_t n_fixing_up = 0; // How many are working on fixup slash count.
static struct vec *fixup_dirs;

static pthread_mutex_t fixup_mutex = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t fixup_cond = PTHREAD_COND_INITIALIZER;

static size_t countslash(const char *s) {
size_t count = 0;
while (*s) {
if ((*s) == '/')
count++;
s++;
}
return count;
}

static int compare_number_of_slashes(const struct vecitem *a, const struct vecitem *b) {
size_t as = countslash(a->fullname);
size_t bs = countslash(b->fullname);
if (as < bs) {
return -1;
}
if (as > bs) {
return +1;
}
return 0;
}

static void* fixup_runner(void *ignore __attribute__((__unused__))) {
pthread_mutex_lock(&fixup_mutex);
while (1) {
while (1
&& vec_size(fixup_dirs)
&& n_fixing_up
&& countslash(vec_peek(fixup_dirs)->fullname) < fixup_slash_count) {
// If n_dirs_to_fixup becomes zero, then the
// fixup_slash_count will also have changed (and a
// broadcast happens when the fixup_slash_count
// changes).
//
// If n_fixing_up becomes zero then we really only care if
// the fixup_slash_count changes (in whichcase a broadcast
// happened) or the n_dirs_to_fixup goes to 0 (in which
// case we broadcast).
//
// If the fixup_slash_count changes we broadcast.
pthread_cond_wait(&fixup_cond, &fixup_mutex);
}
if (vec_size(fixup_dirs) == 0) {
break; // No more work to do
}
size_t sc = countslash(vec_peek(fixup_dirs)->fullname);
if (sc == fixup_slash_count) {
n_fixing_up++;
struct vecitem *dtf = vec_pop(fixup_dirs);
spew(SPEW_DEBUG, "working on %s", dtf->fullname);
pthread_mutex_unlock(&fixup_mutex);
if (arg_preserve && dtf->uid != SIZE_MAX) {
int r __attribute__((unused)) =
chown(dtf->fullname, (uid_t)dtf->uid, (gid_t)dtf->gid);
// Don't complain if this goes wrong
}
{
int r __attribute__((unused)) =
chmod(dtf->fullname, (mode_t)dtf->mode);
// Don't complain if this goes wrong
}
if (dtf->mtime) {
struct timespec times[2] = {{.tv_nsec=UTIME_OMIT},
{.tv_sec = (time_t)dtf->mtime, .tv_nsec=0}};
int r = utimensat(AT_FDCWD, dtf->fullname, times, AT_SYMLINK_NOFOLLOW);
assert(r == 0);
}
FREE(dtf->fullname);
FREE(dtf);
pthread_mutex_lock(&fixup_mutex);
n_fixing_up--;
if (n_fixing_up == 0 && vec_size(fixup_dirs) == 0) {
pthread_cond_broadcast(&fixup_cond);
}
} else {
assert(n_fixing_up == 0);
fixup_slash_count = sc;
{ // debug code
size_t count = 0;
for (size_t i = 0; i < vec_size(fixup_dirs); i++) {
if (countslash(vec_fetch(fixup_dirs, i)->fullname) == sc) count++;
}
spew(SPEW_DEBUG, "sc=%ld (count=%lu)", sc, count);
}
pthread_cond_broadcast(&fixup_cond);
}
}
pthread_mutex_unlock(&fixup_mutex);
return NULL;
}

static void Close(int fd) {
int r = close(fd);
assert(r == 0);
}

static void Dup2(int oldfd, int newfd) {
int r = dup2(oldfd, newfd);
if (r != newfd) {
fprintf(stderr, "dup2(%d %d) -> %d --> errno=%d (%s)\n",
oldfd, newfd, r, errno, strerror(errno));
spew(SPEW_DEBUG, "dup2(%d %d) -> %d --> errno=%d (%s)",
oldfd, newfd, r, errno, strerror(errno));
}
}


enum { READ_END = 0, WRITE_END = 1 };

static int Pipefrom(int fd, const char *execarg, ...) {
size_t n = 0;
char **MALLOC_N(argv, 1);
va_list ap;
va_start(ap, execarg);
while (1) {
REALLOC(argv, n+1);
char *s = va_arg(ap, char *);
argv[n++] = s;
if (s == NULL) break;
}
va_end(ap);
int pipefd[2];
{
int r = pipe(pipefd);
assert(r == 0);
}
if (fork() == 0) {
Dup2(fd, STDIN_FILENO);
Close(fd);
Dup2(pipefd[WRITE_END], STDOUT_FILENO);
Close(pipefd[WRITE_END]);
Close(pipefd[READ_END]);
execvp(execarg, argv);
assert(0);
} else {
Close(fd);
Close(pipefd[WRITE_END]);
FREE(argv);
return pipefd[READ_END];
}
}

static int Pipeto(int fd, const char *execarg, ...) {
size_t n = 0;
char **MALLOC_N(argv, n);
va_list ap;
va_start(ap, execarg);
while (1) {
REALLOC(argv, n+1);
char *s = va_arg(ap, char *);
argv[n++] = s;
if (s == NULL) break;
}
va_end(ap);
int pipefd[2];
{
int r = pipe(pipefd);
assert(r == 0);
}
if (fork() == 0) {
Dup2(fd, STDOUT_FILENO);
Close(fd);
Dup2(pipefd[READ_END], STDIN_FILENO);
Close(pipefd[WRITE_END]);
Close(pipefd[READ_END]);
execvp(execarg, argv);
assert(0);
} else {
Close(fd);
Close(pipefd[READ_END]);
return pipefd[WRITE_END];
}
}

struct stringset *directories_i_know_about;

static void dirname_of_pathname(const char *pathname, char *dirname) {
// find the destination directory name from the full pathname
size_t filelen = strlen(pathname) + 1;
strncpy(dirname, pathname, filelen);
const size_t nl = strlen(dirname);
if (nl != 0
&& dirname[nl-1] == '/')
{
dirname[nl-1] = 0;
}
char *lastslash = strrchr(dirname, '/');
if (lastslash) {
*(lastslash+1) = 0;
} else {
dirname[0] = 0; // no slashes, so use the empty dir
}
}

static void ensure_directory_exists(const char *dirname, int implicitly_create)
// implicitly_create is 1 for directories that are not explicitly created,
// so we should simply mkdir using the default permissions implied by
// whatever umask is.
{
if (dirname[0] &&
!stringset_contains(directories_i_know_about, dirname) != 0)
{
// Don't know about it.
size_t dirlen = strlen(dirname)+1;
char *parentdir = malloc(dirlen);
memset(&parentdir[0], 0, dirlen);
dirname_of_pathname(dirname, parentdir);
ensure_directory_exists(parentdir, 1);
if (implicitly_create) {
int r = mkdir(dirname, 0777);
if (r != 0 && errno != EEXIST) {
fprintf(stderr, "Trying to mkdir %s\n", dirname);
perror("mkdir");
fprintf(stderr, "I'll try to continue\n");
spew(SPEW_ERROR, "mkdir %s failed errno=%d (%s)", dirname, errno, strerror(errno));
}
} else {
struct fwobject *b = mk_fwobject('5', dirname, NULL, 0, 0, 0, 0, 0, NULL);
fwg_add2(fwg, dirname, b, sizeof(*b) + strlen(dirname),
dirname, parentdir);
}
stringset_insert(directories_i_know_about, dirname);
if(parentdir) {
FREE(parentdir);
}
}
}

static size_t extracttar(void)
// Return the error count.
{
size_t errcount = 0;
int input_fd = arg_named_input
? open(arg_named_input, O_RDONLY | O_CLOEXEC)
: dup(STDIN_FILENO);
if(input_fd < 0) {
fprintf(stderr, "No such tar file exists : %s\n", arg_named_input);
errcount++;
return errcount;
}

int pv_fd = arg_show_progress
? Pipefrom(input_fd, "pv", "pv", NULL)
: input_fd;
assert(pv_fd >= 0);

int unzipped_input = arg_gunzip
? Pipefrom(pv_fd, "gunzip", "gunzip", "-c", NULL)
: pv_fd;
assert(unzipped_input);

FILE *input_file = fdopen(unzipped_input, "r");

if (arg_change_dir) {
int r = chdir(arg_change_dir);
if (r != 0) {
fprintf(stderr, "Couldn't change to dir %s\n", arg_change_dir);
perror("chdir");
spew(SPEW_ERROR, "chdir %s failed errno=%d (%s)", arg_change_dir, errno, strerror(errno));
exit(1);
}
}

fixup_dirs = mk_vec();
directories_i_know_about = mk_stringset();
pinit();
start_runners();
int saw_end = 0;
size_t count = 0;

char prev_prev_typeflag = 0;
char prev_typeflag = 0;

// For long names the encoding is to say (with 'L') "next one or more
// 512-byte block contains the name of the file for the following
// record". Similarly to say with 'K' that the next one or more 512-byte
// block contains the linkname for the file in the following
// record". And if there is an L followed by a K then they both
// modify the following record.
// Here the long_name_from_prev is supposed to store the full name of
// the file for encoding = 'L' which can span across multiple blocks.

int has_name_from_prev = 0;
char *long_name_from_prev = NULL;
int has_linkname_from_prev = 0;
struct posix_header linkname_from_prev;
linkname_from_prev.name[0] = 0;

size_t regcount = 0, linkcount = 0, symlinkcount = 0, dircount = 0;

size_t n_records_seen = 0;

while (1) {

if (prev_typeflag != 'L' && prev_typeflag != 'K') {
has_name_from_prev = 0;
has_linkname_from_prev = 0;
}

char *fullname;
char linkname[600];
struct posix_header h;
{
static struct posix_header zero;
h = zero;
size_t r = fread(&h, 1, sizeof(h), input_file);
if (r == 0 && feof(input_file)) {
break;
}
if (n_records_seen == 0 &&
memcmp(h.magic, "ustar", 5) != 0)
{
fprintf(stderr, "Input does not appear to be a tar file.\n");
spew(SPEW_ERROR, "Input does not appear to be a tar file.");
if (0 == memcmp(h.name,
(unsigned char[]){0x1f, 0x8b, 0x08}, 3))
{
fprintf(stderr, "Input appears to be gzipped, ");
spew(SPEW_ERROR, "Input appears to be gzipped, ");
if (arg_gunzip) {
fprintf(stderr, "which is confusing since you specified -z\n");
spew(SPEW_ERROR, "which is confusing since you specified -z");
} else {
fprintf(stderr, "without specifying -z, e.g., as\n"
" %s xzf file.tar.gz\n", cmd);
spew(SPEW_ERROR, "without specifying -z, e.g., as\n"
" %s xzf file.tar.gz", cmd);
}
} else if (0 ==
memcmp(
h.name,
(unsigned char[]){0xFD, '7', 'z', 'X', 'Z'},
5)) {
fprintf(stderr,
"Input appears to be xz compressed,"
" which %s cannot (yet) decompress.\n",
cmd);
fprintf(stderr,
"Maybe you can use an external decompressor,"
" e.g., as\n xzcat file.tar.xz|%s xf -\n",
cmd);
spew(SPEW_ERROR,
"Input appears to be xz compressed,"
" which %s cannot (yet) decompress.",
cmd);
spew(SPEW_ERROR,
"Maybe you can use an external decompressor,"
" e.g., as\n xzcat file.tar.xz|%s xf -",
cmd);
} else {
fprintf(stderr, "Maybe it needs to be decompressed.\n");
spew(SPEW_DEBUG, "Maybe it needs to be decompressed.");
}
exit(1);
}
assert(r == sizeof(h));
n_records_seen++;
}
count++;
{
char *hname = h.name;
size_t hname_limit = sizeof(h.name);
if (has_name_from_prev) {
hname = long_name_from_prev;
spew(SPEW_DEBUG, "prev record name = %s", hname);
hname_limit = strlen(long_name_from_prev);
} else {
if(long_name_from_prev) {
FREE(long_name_from_prev);
}
}
// The tar docs say that linkname is NUL terminated, but it's not.
if (has_linkname_from_prev) {
snprintf(linkname, sizeof(linkname), "%.*s",
(int)512, linkname_from_prev.name);
spew(SPEW_DEBUG, "p linkname=%s (%ld)", linkname, sizeof(linkname_from_prev.name));
} else {
snprintf(linkname, sizeof(linkname), "%.*s",
(int)sizeof(h.linkname), h.linkname);
spew(SPEW_DEBUG, "h linkname=%s", linkname);
}
if (h.prefix[0]) {
fullname = malloc(hname_limit + sizeof(h.prefix) + 1);
int flen = snprintf(fullname, hname_limit + sizeof(h.prefix) + 1, "%.*s/%.*s",
(int)sizeof(h.prefix), h.prefix,
(int)hname_limit, hname);
assert((size_t)flen <= (hname_limit + sizeof(h.prefix)));
} else {
fullname = malloc(hname_limit + 1);
int flen = snprintf(fullname, hname_limit + 1, "%.*s",
(int)hname_limit, hname);
assert((size_t)flen <= hname_limit);
}
spew(SPEW_DEBUG, "prefix=%s fullname=%s", h.prefix, fullname);
}

int exclude_me = matches_excludes(fullname);
spew(SPEW_DEBUG, "Exclude %s --> %d", fullname, exclude_me);

size_t dirlen = strlen(fullname)+1;
char *dirname = malloc(dirlen);
memset(dirname, 0, strlen(fullname));
dirname_of_pathname(fullname, dirname);
spew(SPEW_DEBUG, "dirname=%s sizeof(dirname)=%lu fullname=%s",
dirname, sizeof(dirname), fullname);
if (!exclude_me)
ensure_directory_exists(dirname, 1);

assert(h.pad[0]==0);
size_t size_to_write = parse_octal_number(h.size, sizeof(h.size));
size_t size_to_read = ((size_to_write+511)/512)*512;
if (size_to_write) {
spew(SPEW_DEBUG, "Write size=%lu", size_to_write);
}
if (size_to_read) {
spew(SPEW_DEBUG, "Read size=%lu ", size_to_read);
}
spew(SPEW_DEBUG, "type=%c", *h.typeflag);

size_t mode = parse_octal_number(h.mode, sizeof(h.mode));

if (*h.typeflag == '5') {
// Mkdir.
if (!exclude_me) {
dircount++;
ensure_directory_exists(fullname, 0);
struct vecitem *MALLOC(dtf);
*dtf = (struct vecitem){
strdup(fullname),
mode,
parse_octal_number(h.uid, sizeof(h.uid)),
parse_octal_number(h.gid, sizeof(h.gid)),
parse_octal_number(h.mtime, sizeof(h.mtime))
};
vec_push(fixup_dirs, dtf);
}
} else if (*h.typeflag == '0') {
regcount++;
// Regular file.

spew(SPEW_DEBUG, "insert regular file %s size=%lu mode=0%lo",
fullname, size_to_write, mode);
if (exclude_me) {
size_t bufsize = 16ul * 1024ul * 1024ul;
char *bytes = malloc(bufsize);
while (size_to_read > 0) {
size_t read_now = smin(size_to_read, bufsize);
size_t s = do_read(input_file, bytes, read_now);
if (s != read_now) {
fprintf(stderr, "Tried to read %ld bytes, got %ld\n", read_now, s);
spew(SPEW_WARN, "Tried to read %ld bytes, got %ld", read_now, s);
}
assert(s == read_now);
size_to_read -= read_now;
}
free(bytes);
} else if (size_to_read * 2 > memory_budget_MiB * MiB) {
int done = 0;
struct fwobject *b = mk_fwobject_read_from_input(
*h.typeflag,
fullname,
size_to_write,
mode,
parse_octal_number(h.uid, sizeof(h.uid)),
parse_octal_number(h.gid, sizeof(h.gid)),
parse_octal_number(h.mtime, sizeof(h.mtime)),
input_file,
size_to_read,
&done);
fwg_add2(fwg, fullname,
b, sizeof(*b) + strlen(fullname),
fullname, dirname);
pthread_mutex_lock(&fwmutex);
while (!done) {
pthread_cond_wait(&fwcond, &fwmutex);
}
pthread_mutex_unlock(&fwmutex);
spew(SPEW_DEBUG, "main proceeding");
} else {
// malloc and free are fine with size_to_read==0.
char *bytes = malloc(size_to_read);
spew(SPEW_DEBUG, "malloc(%zu) = %p", size_to_read, bytes);
if (size_to_read) {
size_t s = do_read(input_file, bytes, size_to_read);
assert(s == size_to_read);
}
struct fwobject *b = mk_fwobject(*h.typeflag,
fullname,
NULL,
size_to_write,
mode,
parse_octal_number(h.uid, sizeof(h.uid)),
parse_octal_number(h.gid, sizeof(h.gid)),
parse_octal_number(h.mtime, sizeof(h.mtime)),
bytes);
fwg_add2(fwg, fullname,
b, sizeof(*b) + strlen(fullname) + size_to_write,
fullname, dirname);
}
} else if (*h.typeflag == 0) {
if (saw_end == 0) {
saw_end = 1;
} else {
if (dirname) {
FREE(dirname);
}
if (fullname) {
FREE(fullname);
}
break;
}
} else if (*h.typeflag == 'g') {
spew(SPEW_DEBUG, "global extended header");
char *bytes = malloc(size_to_read);
assert(bytes);
size_t s = do_read(input_file, bytes, size_to_read);
assert(s == size_to_read);
free(bytes);
} else if (*h.typeflag == '1') {
// We have to wait on the fullname and the linkname. It's
// possible that the directories are the same.
linkcount++;
assert(linkname[0] && fullname[0]);
// If the fullname and the linkname are the same, then
// don't do anything. That might not be right if the new
// link has different permissions or something.
if (!exclude_me &&
strcmp(fullname, linkname) != 0)
{
struct fwobject *b = mk_fwobject(*h.typeflag,
fullname,
linkname,
0, 0, 0, 0, 0, NULL);
spew(SPEW_DEBUG, "adding '%c'", *h.typeflag);
fwg_add3(fwg, fullname,
b, sizeof(*b) + strlen(fullname) + strlen(linkname),
fullname, dirname, linkname);
}
} else if (!exclude_me &&
(*h.typeflag == '2' // symlink
|| *h.typeflag == '6' // fifo
)) {
if (*h.typeflag == 2) symlinkcount++;
struct fwobject *b = mk_fwobject(*h.typeflag,
fullname,
*h.typeflag == '2' ? linkname : NULL,
0,
mode, 0, 0, 0,
NULL);
spew(SPEW_DEBUG, "adding '%c'", *h.typeflag);
fwg_add2(fwg, fullname, b, sizeof(*b) + strlen(fullname) + strlen(linkname),
fullname, dirname);
} else if (*h.typeflag == 'L') {
if (0 && prev_typeflag != '5' && prev_typeflag != '0') {
fprintf(stderr, "L: prev_typeflag = %c\n", prev_typeflag);
spew(SPEW_DEBUG, "L: prev_typeflag = %c", prev_typeflag);
}
long_name_from_prev = malloc(size_to_read);
size_t r = fread(long_name_from_prev, 1, size_to_read, input_file);
assert(r == size_to_read);
spew(SPEW_DEBUG, "Long name: name=%s\n", long_name_from_prev);
has_name_from_prev = 1;
} else if (*h.typeflag == 'K') {
if (0 && prev_typeflag != '5' && prev_typeflag != '0') {
fprintf(stderr, "K: prev_typeflag = %c\n", prev_typeflag);
spew(SPEW_DEBUG, "K: prev_typeflag = %c", prev_typeflag);
}
size_t r = fread(&linkname_from_prev, 1, sizeof(linkname_from_prev), input_file);
assert(r == sizeof(linkname_from_prev));
has_linkname_from_prev = 1;
} else {
fprintf(stderr, "h.typeflag=%c (prev_typeflag=%c) (prev_prev_typeflag=%c)\n", *h.typeflag, prev_typeflag, prev_prev_typeflag);
spew(SPEW_DEBUG, "h.typeflag=%c (prev_typeflag=%c) (prev_prev_typeflag=%c)", *h.typeflag, prev_typeflag, prev_prev_typeflag);
debug_print_posix_header(&h);
abort();
}
prev_prev_typeflag = prev_typeflag;
prev_typeflag = *h.typeflag;
if (dirname) {
FREE(dirname);
}
if (fullname) {
FREE(fullname);
}
}
fwg_end_of_nodes(fwg);
runners_finish();
if (vec_size(fixup_dirs)) {
const size_t time_fixup = 0;
struct timespec start_fixup, end_fixup;
if (time_fixup) {
clock_gettime(CLOCK_MONOTONIC, &start_fixup);
fprintf(stderr, "Fixing up, there are %lu dirs\n", vec_size(fixup_dirs));
spew(SPEW_DEBUG, "Fixing up, there are %lu dirs", vec_size(fixup_dirs));
}
vec_sort(fixup_dirs, compare_number_of_slashes);
fixup_slash_count = countslash(vec_peek(fixup_dirs)->fullname);
pthread_t *MALLOC_N(fthreads, n_threads);
for (size_t i = 0; i < n_threads; i++) {
pthread_create(&fthreads[i], NULL, fixup_runner, NULL);
}
for (size_t i = 0; i < n_threads; i++) {
void *v;
pthread_join(fthreads[i], &v);
}
FREE(fthreads);
if (time_fixup) {
clock_gettime(CLOCK_MONOTONIC, &end_fixup);;
fprintf(stderr, "Fixup took %6.3fs\n", (double)(end_fixup.tv_sec - start_fixup.tv_sec) + 1e-9*(double)(end_fixup.tv_nsec - start_fixup.tv_nsec));
spew(SPEW_DEBUG, "Fixup took %6.3fs", (double)(end_fixup.tv_sec - start_fixup.tv_sec) + 1e-9*(double)(end_fixup.tv_nsec - start_fixup.tv_nsec));
}
}
// In some cases the dirs_to_fixup are still there (e.g., if we aren't preserving)
while (1) {
struct vecitem *item = vec_pop(fixup_dirs);
if (item == NULL) {
break;
}
FREE(item->fullname);
FREE(item);
}
pdestroy();
fclose(input_file);
directories_i_know_about = stringset_destroy(directories_i_know_about);
fixup_dirs = vec_destroy(fixup_dirs);

spew(SPEW_DEBUG, "regcount=%ld linkcount=%ld symlinkcount=%ld dircount=%lu", regcount, linkcount, symlinkcount, dircount);
while (1) {
int status;
int r = wait(&status);
if (r == -1 && errno == ECHILD) {
break;
}
if (r == -1) {
fprintf(stderr, "%s: waiting on child %s\n", cmd, strerror(errno));
spew(SPEW_DEBUG, "%s: waiting on child %s", cmd, strerror(errno));
errcount++;
break;
}
if (!WIFEXITED(status)) {
fprintf(stderr, "%s: Child did not return normally\n", cmd);
spew(SPEW_ERROR, "%s: Child did not return normally", cmd);
errcount++;
} else if (WEXITSTATUS(status) != 0) {
fprintf(stderr, "%s: Child returned status %d\n", cmd, WEXITSTATUS(status));
spew(SPEW_ERROR, "%s: Child returned status %d", cmd, WEXITSTATUS(status));
errcount++;
}
}
if (errcount == 0 && n_records_seen == 0) {
fprintf(stderr, "%s: This does not look like a tar archive\n", cmd);
spew(SPEW_ERROR, "%s: This does not look like a tar archive", cmd);
errcount++;
}
return errcount;
}

// For createtar
static FILE *output_file;
static size_t n_written = 0;
static pthread_mutex_t emit_mutex = PTHREAD_MUTEX_INITIALIZER;

static unsigned char ph_string[512];
static size_t ph_off;

static void e_char(unsigned char c) {
ph_string[ph_off++] = c;
}

static void e_string_short(const char *string, size_t len) {
assert(ph_off + len <= sizeof(ph_string));
size_t slen = strlen(string);
if(slen < len) {
memcpy(ph_string + ph_off, string, slen);
}
else {
memcpy(ph_string + ph_off, string, len);
}
while (slen < len) {
ph_string[ph_off + slen++] = 0;
}
ph_off += len;
}

static void e_string(const char *string, size_t len) {
assert(strlen(string) <= len);
e_string_short(string, len);
}

static void e_number(size_t number, size_t len) {
int r = snprintf((char*)ph_string+ph_off, 512-ph_off, "%.*zo",
(int)len-1, number);
assert(r >= 0);
assert(ph_off + (size_t)r < 512);
if ((size_t)r > len-1) {
// It's too big, and must be represented in binary.
assert((len > sizeof(size_t)) || (0 == (number >> (8 * len - 1))));
for (size_t i = len; i > 0; i--) {
unsigned char extra_bits = (i == 1) ? 0x80u : 0;
unsigned char the_byte = (unsigned char)(number & 0xffu);
ph_string[ph_off + i - 1] = extra_bits | the_byte;
number >>= 8;
}
}
ph_off += len;
}

static void e_checksum_blanks(void) {
for (size_t i = 0; i < 8; i++) e_char(' ');
}


static void e_prepare_checksum(void) {
size_t checksum = 0;
for (size_t i = 0; i < 512; i++) checksum += (unsigned char)ph_string[i];
snprintf((char*)ph_string+148, 8, "%06zo", checksum);
}

static void emit_longlink(const char *fname, unsigned char typ) {
e_string("././@LongLink", 100);
e_number(0644, 8);
e_number(0, 8);
e_number(0, 8);
e_number(strlen(fname)+1, 12);
e_number(0, 12);
e_checksum_blanks();
e_char(typ);
e_string("", 100);
e_string("ustar ", 8);
e_string("root", 32);
e_string("root", 32);
e_string("", 8+8+155+12);
assert(ph_off == 512);
e_prepare_checksum();
fwrite(ph_string, 1, ph_off, output_file);
n_written += ph_off;
ph_off = 0;
fputs(fname, output_file);
n_written += strlen(fname);
size_t i = strlen(fname);
if(!(i % 512)) {
putc(0, output_file);
n_written++;
i++;
}
for (; i % 512; i++) {
putc(0, output_file);
n_written++;
}
}

static void emit_header_locked(const char *fname,
struct stat *statbuf,
const char *link_target,
unsigned char link_typeflag) {
// To get it going, assume that the fname is less than 100 chars
ph_off = 0;
if (link_target) {
if (strlen(link_target) > 100) {
emit_longlink(link_target, 'K');
}
}
if (S_ISDIR(statbuf->st_mode)) {
char *output_fname;
int r = asprintf(&output_fname, "%s/", fname);
assert(r >= 0);
if (strlen(fname)+1 > 100) {
emit_longlink(output_fname, 'L');
e_string_short(output_fname, 100);
} else {
e_string(output_fname, 100);
}
free(output_fname);
} else {
if (strlen(fname) > 100) {
emit_longlink(fname, 'L');
e_string_short(fname, 100);
} else {
e_string(fname, 100);
}
}
e_number(statbuf->st_mode & 07777, 8);
e_number(statbuf->st_uid, 8);
e_number(statbuf->st_gid, 8);
assert(statbuf->st_size >= 0);
if (S_ISLNK(statbuf->st_mode) || S_ISDIR(statbuf->st_mode) || link_target) {
e_number(0, 12);
} else {
e_number((size_t)statbuf->st_size, 12);
}
e_number((size_t)statbuf->st_mtim.tv_sec, 12);
e_checksum_blanks();
if (link_typeflag) {
e_char(link_typeflag);
} else if (S_ISREG(statbuf->st_mode)) {
e_char('0');
} else if (S_ISLNK(statbuf->st_mode)) {
e_char('2');
} else if (S_ISDIR(statbuf->st_mode)) {
e_char('5');
} else if (S_ISFIFO(statbuf->st_mode)) {
e_char('6');
} else {
abort();
}
if (S_ISLNK(statbuf->st_mode)) {
assert(link_target);
if (strlen(link_target) <= 100) {
e_string(link_target, 100);
} else {
e_string_short(link_target, 100);
}
} else {
if (link_target && strlen(link_target) <= 100) {
e_string(link_target, 100);
} else {
e_string("", 100);
}
}
e_string("ustar ", 8);
{ // uname
struct passwd *pw = getpwuid(statbuf->st_uid);
if (pw == NULL) {
e_string("", 32);
} else {
e_string(pw->pw_name, 32);
}
}
{ // gname
struct group *g = getgrgid(statbuf->st_gid);
if (g == NULL) {
e_string("", 32);
} else {
e_string(g->gr_name, 32);
}
}
e_string("", 8); // devmajor
e_string("", 8); // devminor
e_string("", 155);
e_string("", 12);
assert(ph_off == 512);
e_prepare_checksum();
fwrite(ph_string, 1, ph_off, output_file);
ph_off = 0;
n_written += 512;
}

static void emit_data_locked(const char *fname, size_t size) {
FILE *f = fopen(fname, "r");
if (f == NULL) {
return;
}
const size_t size_limit = 1024*1024u; // bigger than this, we grab the lock early
char *buf=malloc(size_limit);
size_t remaining_size = size;
while (remaining_size) {
size_t read_now = remaining_size > size_limit ? size_limit : remaining_size;
size_t actual_read = fread(buf, 1, read_now, f);
size_t actual_write = fwrite(buf, 1, actual_read, output_file);
n_written += actual_read;
remaining_size -= actual_write;
}
while (n_written % 512) {
n_written++;
putc(' ', output_file);
}
fclose(f);
FREE(buf);
}

struct inodemap *inset;

static int maybe_emit_link_locked(const char *fname, struct stat *statbuf) {
void const *value;
if (inodemap_lookup(inset, statbuf->st_ino, statbuf->st_dev, &value)) {
emit_header_locked(fname, statbuf, value, '1');
return 1;
} else {
inodemap_insert(inset,statbuf->st_ino, statbuf->st_dev, strdup(fname));
return 0;
}
}

static void emit(const char *fname, struct stat *statbuf) {
const off_t size_limit = 1024*1024u; // bigger than this, we grab the lock early
if (S_ISREG(statbuf->st_mode)) {
if (statbuf->st_size > size_limit) {
pthread_mutex_lock(&emit_mutex);
if (!maybe_emit_link_locked(fname, statbuf)) {
emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0);
emit_data_locked(fname, (size_t)statbuf->st_size);
}
pthread_mutex_unlock(&emit_mutex);
} else {
assert(statbuf->st_size >= 0);
char *buf = malloc((size_t)statbuf->st_size);
FILE *f = fopen(fname, "r");
if (f == NULL) {
FREE(buf);
return;
}
size_t s = statbuf->st_size == 0 ? 0 : do_read(f, buf, (size_t)statbuf->st_size);
assert(s == (size_t)statbuf->st_size);
fclose(f);
pthread_mutex_lock(&emit_mutex);
if (!maybe_emit_link_locked(fname, statbuf)) {
emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0);
fwrite(buf, 1, s, output_file);
size_t target = ((s+511)/512)*512;
for (size_t i = s; i < target; i++) {
putc(0, output_file);
}
n_written += target;
}
pthread_mutex_unlock(&emit_mutex);
FREE(buf);
}
} else if (S_ISLNK(statbuf->st_mode)) {
assert(statbuf->st_size >= 0);
char *buf = malloc((size_t)statbuf->st_size + 2);
ssize_t r = readlink(fname, buf, (size_t)statbuf->st_size);
assert(r == statbuf->st_size);
buf[statbuf->st_size] = 0;
pthread_mutex_lock(&emit_mutex);
if (!maybe_emit_link_locked(fname, statbuf)) {
emit_header_locked(fname, statbuf, buf, '2');
}
pthread_mutex_unlock(&emit_mutex);
FREE(buf);
} else {
pthread_mutex_lock(&emit_mutex);
if (!maybe_emit_link_locked(fname, statbuf)) {
emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0);
}
pthread_mutex_unlock(&emit_mutex);
}
}

static magic_t ctar_magic = "ctar magic";

struct ctarframe {
magic_t *magic;
struct vector_of_strings *vs;
size_t inlet_sum;
size_t sum;
};

static struct ctarframe *mk_ctarframe(struct vector_of_strings *vs) {
struct ctarframe *MALLOC(cf);
*cf = (struct ctarframe){.magic = &ctar_magic,
.vs = vs,
.inlet_sum = 0,
.sum = 0};
return cf;
}

static struct ctarframe *ctarframe_free(struct ctarframe *cf) {
vector_of_strings_destroy(cf->vs);
FREE(cf);
return NULL;
}

static struct ctarframe *ctarframe_cast(void *app_frame) {
struct ctarframe *cf = app_frame;
assert(cf->magic == &ctar_magic);
return cf;
}

static void ctarfun_return_inlet(void *parent_app_frame, size_t result) {
struct ctarframe *cf = ctarframe_cast(parent_app_frame);
cf->inlet_sum += result;
}

static size_t ctar_error_count = 0;
static pthread_mutex_t ctar_error_count_mutex = PTHREAD_MUTEX_INITIALIZER;
static void bump_ctar_error_count(void) {
pthread_mutex_lock(&ctar_error_count_mutex);
ctar_error_count++;
pthread_mutex_unlock(&ctar_error_count_mutex);
}


static struct frame *ctarfun2(struct frame *frame, void *app_frame);

static struct frame*ctarfun(struct frame *frame, void *app_frame) {
struct ctarframe *cf = ctarframe_cast(app_frame);
char *fname;
while ((fname = vector_of_strings_pop(cf->vs))) {
if (!matches_excludes(fname)) {
struct stat statbuf;
{
int r = lstat(fname, &statbuf);
if (r != 0) {
fprintf(stderr, "%s: %s: Cannot stat: %s\n", cmd, fname, strerror(errno));
spew(SPEW_ERROR, "%s: %s: Cannot stat: %s", cmd, fname, strerror(errno));
bump_ctar_error_count();
continue;
}
}
cf->sum++;
emit(fname, &statbuf);
if (S_ISDIR(statbuf.st_mode)) {
struct vector_of_strings *vs = mk_vector_of_strings();
DIR *dir = opendir(fname);
assert(dir);
struct dirent *entry;
while ((entry = readdir(dir))) {
if (strcmp(entry->d_name, ".") == 0) continue;
if (strcmp(entry->d_name, "..") == 0) continue;
char *path = pathcat(fname, entry->d_name);
vector_of_strings_push(vs, path);
FREE(path);
}
free(fname);
closedir(dir);
struct ctarframe *subframe = mk_ctarframe(vs);
return ctarfun(
sched_spawn(frame, ctarfun, subframe, ctarfun_return_inlet),
subframe);
}
}
free(fname);
}
return sched_sync(frame, ctarfun2);
}

static struct frame *ctarfun2(struct frame *frame, void *app_frame) {
struct ctarframe *cf = ctarframe_cast(app_frame);
size_t sum = cf->inlet_sum + cf->sum;
cf = ctarframe_free(cf);
return sched_return(frame, sum);
}

static void do_createtar(struct vector_of_strings *vs) {
struct ctarframe *cf = mk_ctarframe(vs);
size_t n = prun(n_threads, ctarfun, cf);
spew(SPEW_INFO, "Tarred %zu objects", n);
}

static size_t createtar(struct vector_of_strings *vs) {
ctar_error_count = 0;
int output_fd = arg_named_input
? open(arg_named_input, O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, 0666)
: dup(STDOUT_FILENO);
assert(output_fd >= 0);
int unzipped_output = arg_gunzip
? Pipeto(output_fd, "gzip", "gzip", "-c", NULL)
: output_fd;
output_file = fdopen(unzipped_output, "w");
if (arg_change_dir) {
int r = chdir(arg_change_dir);
if (r != 0) {
fprintf(stderr, "Couldn't change to dir %s\n", arg_change_dir);
perror("chdir");
spew(SPEW_ERROR, "Couldn't change to dir %s errno=%d (%s)", arg_change_dir, errno, strerror(errno)) ;
exit(1);
}
}
do_createtar(vs);
spew(SPEW_INFO, "n-written=%zu\n", n_written);
while (n_written % (20*512u)) {
// Tar puts at least 2 512-byte blocks at the end, and enough to add up to a multiple of 20 512 blocks.
putc(0, output_file);
n_written++;
}
fclose(output_file);
return ctar_error_count++;
}

static void destroy_string(void *sv) {
FREE(sv);
}

int main (int argc, char *argv[]) {
int ret = 0;
find_open_fds_at_start();
we_are_root = geteuid () == ROOT_UID;
if (we_are_root) arg_preserve = 1;
struct vector_of_strings *vs = mk_vector_of_strings();
excludes = mk_vector_of_strings();

#ifdef SPEW_ENABLED
size_t spew_level = SPEW_DEBUG;

spew_config(argc, argv,
"partar_log",
/* log roll time in minutes */1440,
/* spew_to_stderr */0,
/*debug=*/0);
spew_set_level(spew_level);
spew(SPEW_INFO, "partar %s, SPLASH!!!", gitrev);
#endif

ret = parse_args(argc, argv, vs);
switch (ret) {
case DONE:
case ERROR:
goto do_exit;
break;
case CONTINUE:
ret = 0;
break;
default:
ret = 1;
goto do_exit;
break;
}
// parsing done and there is work to do!
spew(SPEW_DEBUG, "arg_change_dir = %s", arg_change_dir);
spew(SPEW_DEBUG, "\targ_named_input = %s", arg_named_input);

size_t errcount = 0;
if (tarmode == 'x') {
errcount = extracttar();
vector_of_strings_destroy(vs);
} else if (tarmode == 'c') {
inset = mk_inodemap();
errcount = createtar(vs);
inset = inodemap_destroy(inset, destroy_string);
} else {
assert(0);
}

if (errcount || some_errors) {
fprintf(stderr,
"%s (%s %s): Exiting with failure status due to previous errors\n",
cmd, VERSION, gitrev);
spew(SPEW_ERROR,
"%s (%s %s): Exiting with failure status due to previous errors",
cmd, VERSION, gitrev);
ret = 2;
}

do_exit:
spew_teardown();
fdleak_check();
vector_of_strings_destroy(excludes);
return ret;
}