/* * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. * * Licensed under the Universal Permissive License v 1.0 as shown at * http://oss.oracle.com/licenses/upl * */ /* parallel copy of a directory heirarchy */ #define _GNU_SOURCE #include "inodemap.h" #include "todo.h" #include "malloc.h" #include "putils.h" #include "vector-of-strings.h" #include "version.h" #include #include #include #include #include #include #include #include #include #include #include #include #include // Don't accidentally use pathcat(). Use pathcatn(). char *pathcat(const char *a, const char *b) __attribute__((deprecated)); static char * strdupn(const char *a) { if (a == NULL) return NULL; else return strdup(a); } static char * pathcatn(const char *a, const char *b) { if (b == NULL) return strdupn(a); if (a == NULL) return strdupn(b); char *result = NULL; int r = asprintf(&result, "%s/%s", a, b); assert(r > 0); return result; } enum { default_n_threads = 32 }; // Stuff filled in by parse_args() static struct parsed_args { const char *cmd; size_t n_threads; // All of the following are set by --restore int ignore_times; int one_file_system; int hard_links; int preserve_symlinks; int preserve_perms; int preserve_times; int preserve_owner; int preserve_group; int preserve_devices; int preserve_specials; int delete; struct vector_of_strings *vs; // the arguments struct vector_of_strings *exclude_from; struct vector_of_strings *exclude; struct vector_of_strings *include_from; struct vector_of_strings *include; } pargs = {.n_threads = default_n_threads}; /************************************************************************/ /* Discussion of how it all works. * * Note: we don't have a way to delete extraneous files on the destination. * * We walk the source directory tree in parallel as follows: We have a global * todo list, and we put a todo item that identifies the source and dest into * the todo list. * * A worker grabs an item from the todo list. If the item is a directory, * then use readdir to get all the directory entries. For each directory * entry we create the destination object, and then if there's any additional * work to do (to copy the contents), then we add that work to to the todo * list. * * To copy an object from src to destination while obeying hardlinks: * * Copying srcname to dstname * * Let srcinum be the inode number of the object named by srcname. Look in * src_inodes to see if it's already been created as filename idstname with * inode number idstnum. * * If it's already been created, then we need to make a hard link from * dstname to idstname. * Case 1: There is no object named dstname. Then make the link. * * Case 2: There is an object named dstname, and it has the same inumber * as idstname, then we are done the link is right. * * Case 3: There is an object named dstname but it has a different * inumber, so we must delete the old object (if it's a directory it must * be an empty directory) and then make the lihnk. * */ typedef const char *magic_t; /************************************************************************/ /* Keep track of the topology of inodes. */ /************************************************************************/ magic_t src_inode_value_magic; struct src_inode_value { magic_t *magic; char *fname; dev_t dev; ino_t ino; }; static inline struct src_inode_value * mk_src_inode_value(const char *dst_fname, dev_t dev, ino_t ino) { struct src_inode_value *MALLOC(result); *result = (struct src_inode_value){.magic = &src_inode_value_magic, .fname = strdup(dst_fname), .dev = dev, .ino = ino}; return result; } static struct src_inode_value * src_inode_value_destroy(struct src_inode_value *siv) { assert(siv->magic == &src_inode_value_magic); FREE(siv->fname); FREE(siv); return NULL; } enum { INODE_OWNER_ARRAY_SIZE = 32 }; static struct itopology { pthread_mutex_t map_mutex; // protect the inodemaps during reads and writes. struct inodemap *src_inodes; // A map from src inumber to the // destination filename and the dst // inumber for that destination file. struct inodemap *dst_inodes; // A map from dst inode to a null value // (we are using the map to maintain a // set). pthread_mutex_t inode_owner_array[INODE_OWNER_ARRAY_SIZE]; // Lock based on a hash of the inumber // and is held while the object is being // created. Grab this lock first so // that we can release the map_mutex // while we create the object. } itopology; static void destroy_null(void * x) { assert(x == NULL); } static void destroy_src_inode_value_v(void *xv) { assert(xv); struct src_inode_value *x = xv; assert(x->magic == &src_inode_value_magic); src_inode_value_destroy(x); } static void init_itopology(void) { pthread_mutex_init(&itopology.map_mutex, NULL); itopology.src_inodes = mk_inodemap(); itopology.dst_inodes = mk_inodemap(); for (size_t i = 0; i < INODE_OWNER_ARRAY_SIZE; i++) { pthread_mutex_init(&itopology.inode_owner_array[i], NULL); } } static void deinit_itopology(void) { pthread_mutex_destroy(&itopology.map_mutex); itopology.src_inodes = inodemap_destroy(itopology.src_inodes, destroy_src_inode_value_v); itopology.dst_inodes = inodemap_destroy(itopology.dst_inodes, destroy_null); for (size_t i = 0; i < INODE_OWNER_ARRAY_SIZE; i++) { pthread_mutex_destroy(&itopology.inode_owner_array[i]); } } static void lock_itopology_ownership(dev_t dev, ino_t ino) { uint64_t h = inodemap_hash(dev, ino); pthread_mutex_t *om = &itopology.inode_owner_array[h%INODE_OWNER_ARRAY_SIZE]; pthread_mutex_lock(om); } static void unlock_itopology_ownership(dev_t dev, ino_t ino) { uint64_t h = inodemap_hash(dev, ino); pthread_mutex_t *om = &itopology.inode_owner_array[h%INODE_OWNER_ARRAY_SIZE]; pthread_mutex_unlock(om); } static bool itopology_lookup_src(dev_t dev, ino_t ino, struct src_inode_value const ** sivp) { pthread_mutex_lock(&itopology.map_mutex); void const *sivv; bool result = inodemap_lookup(itopology.src_inodes, dev, ino, &sivv); pthread_mutex_unlock(&itopology.map_mutex); if (result) { assert(sivv); struct src_inode_value const *siv = sivv; assert(siv->magic == &src_inode_value_magic); *sivp = siv; } return result; } static bool itopology_lookup_dst(dev_t dev, ino_t ino) { pthread_mutex_lock(&itopology.map_mutex); void const *np; bool result = inodemap_lookup(itopology.dst_inodes, dev, ino, &np); pthread_mutex_unlock(&itopology.map_mutex); if (result) { assert(np == NULL); } return result; } static void itopology_set(dev_t sdev, ino_t sino, dev_t ddev, ino_t dino, const char *fname) { pthread_mutex_lock(&itopology.map_mutex); inodemap_insert(itopology.dst_inodes, ddev, dino, NULL); inodemap_insert(itopology.src_inodes, sdev, sino, mk_src_inode_value(fname, ddev, dino)); pthread_mutex_unlock(&itopology.map_mutex); } /******************************************************************/ /* todo_item used to create work items for the scheduler. */ /******************************************************************/ static magic_t todo_item_magic; enum todo_stage { TODO_ENSURE_EXISTENCE_AND_POPULATE, TODO_REG_CONTENTS, TODO_DIR_CONTENTS, // Do rm -rf on dst TODO_RM, // only dst defined (not dbuf or src or sbuf) }; struct todo_item { magic_t *magic; enum todo_stage stage; union { struct todo_ensure { const char *src_fprefix; const char *dst_fprefix; char *fsuffix; } ensure; struct todo_reg_or_dir { const char *src_fprefix; const char *dst_fprefix; char *fsuffix; struct stat sbuf; struct stat dbuf; } reg_or_dir; struct todo_rm { const char *dst_fprefix; char *fsuffix; char *notdirname; // just the last part of the fsuffix size_t refcount; struct todo_item *parent; } rm; } u; }; static struct todo_item * mk_todo_item_ensure_existence_and_populate(const char *src_fprefix, const char *dst_fprefix, char *fsuffix) // Ownership of fsuffix gets passed to the new object { struct todo_item *MALLOC(item); *item = (struct todo_item){ .magic = &todo_item_magic, .stage = TODO_ENSURE_EXISTENCE_AND_POPULATE, .u.ensure = {src_fprefix, dst_fprefix, fsuffix}}; return item; } static struct todo_item * mk_todo_item_dir_contents(const char *src_fprefix, const char *dst_fprefix, char *fsuffix, const struct stat *sbuf, const struct stat *dbuf) // Ownership of src and dst get passed to the new object. { struct todo_item *MALLOC(item); *item = (struct todo_item){ .magic = &todo_item_magic, .stage = TODO_DIR_CONTENTS, .u.reg_or_dir = {src_fprefix, dst_fprefix, fsuffix, *sbuf, *dbuf}}; return item; } static struct todo_item * mk_todo_item_reg_contents(const char *src_fprefix, const char *dst_fprefix, char *fsuffix, const struct stat *sbuf, const struct stat *dbuf) // Ownership of src and dst get passed to the new object. { struct todo_item *MALLOC(item); *item = (struct todo_item){ .magic = &todo_item_magic, .stage = TODO_REG_CONTENTS, .u.reg_or_dir = {src_fprefix, dst_fprefix, fsuffix, *sbuf, *dbuf}}; return item; } static struct todo_item * mk_todo_item_rm(const char *dst_fprefix, char *fsuffix, char *notdirname, struct todo_item *parent) { struct todo_item *MALLOC(item); *item = (struct todo_item){ .magic = &todo_item_magic, .stage = TODO_RM, .u.rm = {dst_fprefix, fsuffix, notdirname, 1ul, parent}}; return item; } static struct todo_item * todo_item_destroy(struct todo_item *item) { assert(item->magic == &todo_item_magic); switch (item->stage) { case TODO_ENSURE_EXISTENCE_AND_POPULATE: { struct todo_ensure *e = &item->u.ensure; FREE(e->fsuffix); break; } case TODO_REG_CONTENTS: case TODO_DIR_CONTENTS: { struct todo_reg_or_dir *rd = &item->u.reg_or_dir; FREE(rd->fsuffix); break; } case TODO_RM: { struct todo_rm *rm = &item->u.rm; FREE(rm->fsuffix); FREE(rm->notdirname); break; } default: assert(0); } FREE(item); return NULL; } /******************************************************************/ static int matches_an_exclude(const char *pathname, const char *filename) { if (pathname) { for (size_t i = 0; i < vector_of_strings_size(pargs.exclude); i++) { int r = fnmatch(vector_of_strings_fetch(pargs.exclude, i), pathname, FNM_PATHNAME); if (r == 0) return 1; } } for (size_t i = 0; i < vector_of_strings_size(pargs.exclude); i++) { int r = fnmatch(vector_of_strings_fetch(pargs.exclude, i), filename, FNM_PATHNAME); if (r == 0) return 1; } return 0; } static int matches_an_include(const char *pathname, const char *filename) { // In the absence of --include-from, always include. if (vector_of_strings_size(pargs.include) == 0) { return 1; } if (pathname) { for (size_t i = 0; i < vector_of_strings_size(pargs.include); i++) { int r = fnmatch(vector_of_strings_fetch(pargs.include, i), pathname, FNM_PATHNAME); if (r == 0) return 1; } } for (size_t i = 0; i < vector_of_strings_size(pargs.include); i++) { int r = fnmatch(vector_of_strings_fetch(pargs.include, i), filename, FNM_PATHNAME); if (r == 0) return 1; } return 0; } static void do_ensure_existence_and_populate_item( struct todo_list *tl, const struct todo_ensure *item, struct todo_item *surrounding_item); static void do_dir_contents_item(struct todo_list *tl, const struct todo_reg_or_dir *item, struct todo_item *surrounding_item); static void do_reg_contents_item(struct todo_list *tl, const struct todo_reg_or_dir *item, struct todo_item *surrounding_item); static void do_rm_item(struct todo_list *tl, struct todo_rm *item, struct todo_item *surrounding_item); // This one isn't const because it messes around with refcount. static void do_item(struct todo_list *tl, struct todo_item *item) { // This function now owns item. It's responsible for freeing it. // Pass ownership to one of the subfunctions: switch (item->stage) { case TODO_ENSURE_EXISTENCE_AND_POPULATE: do_ensure_existence_and_populate_item(tl, &item->u.ensure, item); break; case TODO_REG_CONTENTS: do_reg_contents_item(tl, &item->u.reg_or_dir, item); break; case TODO_DIR_CONTENTS: do_dir_contents_item(tl, &item->u.reg_or_dir, item); break; case TODO_RM: do_rm_item(tl, &item->u.rm, item); break; default: assert(0); // unreachable } } pthread_mutex_t todo_rm_refcount_mutex = PTHREAD_MUTEX_INITIALIZER; static void incr_refcount(struct todo_item *item) { if (item == NULL) return; assert(item->magic == &todo_item_magic); assert(item->stage == TODO_RM); pthread_mutex_lock(&todo_rm_refcount_mutex); item->u.rm.refcount++; pthread_mutex_unlock(&todo_rm_refcount_mutex); } static void decr_refcount(struct todo_item *item) { if (item == NULL) return; assert(item->magic == &todo_item_magic); assert(item->stage == TODO_RM); size_t count; pthread_mutex_lock(&todo_rm_refcount_mutex); assert(item->u.rm.refcount > 0); count = --item->u.rm.refcount; pthread_mutex_unlock(&todo_rm_refcount_mutex); if (count == 0) { char *dst = pathcatn(item->u.rm.dst_fprefix, item->u.rm.fsuffix); if (!matches_an_exclude(item->u.rm.fsuffix, item->u.rm.notdirname) && matches_an_include(item->u.rm.fsuffix, item->u.rm.notdirname)) { // Don't worry if this rmdir fails. Sometimes it will fail // because a file inside was excluded (e.g., with --exclude-from), // and that's fine. rmdir(dst); } if (item->u.rm.parent) { decr_refcount(item->u.rm.parent); } todo_item_destroy(item); FREE(dst); } } static void do_rm_item(struct todo_list *tl, struct todo_rm *item, struct todo_item *surrounding_item) { // Pass in the surrounding item so that we can maintain its reference // count. Pass in item so that we we minimize the chance of type errors. assert(surrounding_item->magic == &todo_item_magic); assert(surrounding_item->stage == TODO_RM); assert(&surrounding_item->u.rm == item); struct stat dbuf; char *dst = pathcatn(item->dst_fprefix, item->fsuffix); if (!matches_an_exclude(item->fsuffix, item->notdirname) && matches_an_include(item->fsuffix, item->notdirname)) { MAYBE_ERROR(lstat, dst, &dbuf); if (S_ISDIR(dbuf.st_mode)) { DIR *d = opendir(dst); assert(d); struct dirent *entry; struct vector_of_strings *fnames = mk_vector_of_strings(); struct vector_of_strings *ndnames = mk_vector_of_strings(); while ((entry = readdir(d))) { char *dst_suffix = pathcatn(item->fsuffix, entry->d_name); if (1 && !matches_an_exclude(dst_suffix, entry->d_name) && matches_an_include(dst_suffix, entry->d_name) && strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0 && strcmp(entry->d_name, ".snapshot") != 0) { vector_of_strings_push(fnames, dst_suffix); vector_of_strings_push(ndnames, entry->d_name); } FREE(dst_suffix); } closedir(d); while (vector_of_strings_size(fnames)) { char *dst_suffix = vector_of_strings_pop(fnames); char *ndname = vector_of_strings_pop(ndnames); incr_refcount(surrounding_item); struct todo_item *ti = mk_todo_item_rm(item->dst_fprefix, dst_suffix, ndname, surrounding_item); todo_list_push(tl, ti); } vector_of_strings_destroy(fnames); vector_of_strings_destroy(ndnames); } else { MAYBE_ERROR(unlink, dst); assert(surrounding_item->magic == &todo_item_magic); assert(surrounding_item->stage == TODO_RM); assert(surrounding_item->u.rm.refcount == 1); } } FREE(dst); decr_refcount(surrounding_item); // Don't free the item here, because it might be still alive after // decr_refcount(surrounding_item). } static void do_reg_contents_item(struct todo_list *tl __attribute__((unused)), const struct todo_reg_or_dir *item, struct todo_item *surrounding_item) // Effect: Copy the contents of src to dst, both of which be existing regular // files. We have the stat results for source and dst. We assume that the // check for the case where mtime and size match was already done. { assert(&todo_item_magic == surrounding_item->magic); assert(TODO_REG_CONTENTS == surrounding_item->stage); assert(&surrounding_item->u.reg_or_dir == item); assert(S_ISREG(item->sbuf.st_mode)); // We copy the contents of the file, truncate it, and set the permissions // and mtime. (We assume that the the check for the case where mtime and // size match was done before queuing the item. char *srcname = pathcatn(item->src_fprefix, item->fsuffix); char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); int srcfd = open(srcname, O_RDONLY); maybe_error(srcfd<0, "open", srcname, __FILE__, __LINE__); MAYBE_ERROR(chmod, dstname, 0700); int dstfd = open(dstname, O_WRONLY); maybe_error(dstfd<0, "open", dstname, __FILE__, __LINE__); size_t bufsize=1024*1024; char *buf = malloc(bufsize); size_t totalcount = 0; while (1) { ssize_t rcount = read(srcfd, buf, bufsize); if (rcount == 0) break; maybe_error(rcount<0, "read", srcname, __FILE__, __LINE__); totalcount += (size_t)rcount; char *wbuf = buf; while (rcount > 0) { ssize_t wcount = write(dstfd, wbuf, (size_t)rcount); maybe_error(wcount<0, "write", dstname, __FILE__, __LINE__); rcount -= wcount; wbuf += (size_t)wcount; } } if (item->dbuf.st_size > (off_t)totalcount) { maybe_error(ftruncate(dstfd, (off_t)totalcount), "ftruncate", dstname, __FILE__, __LINE__); } // Must do the chown before setting setuid or setgid. int fchown_ret = fchown(dstfd, item->sbuf.st_uid, item->sbuf.st_gid); maybe_error(fchown_ret, "fchown", dstname, __FILE__, __LINE__); int fchmod_ret = fchmod(dstfd, item->sbuf.st_mode & (0777 | S_ISUID | S_ISGID)); maybe_error(fchmod_ret, "fchmod", dstname, __FILE__, __LINE__); struct timespec times[2] = {item->sbuf.st_atim, item->sbuf.st_mtim}; int futimens_ret = futimens(dstfd, times); maybe_error(futimens_ret, "futimens", dstname, __FILE__, __LINE__); maybe_error(close(srcfd), "close", srcname, __FILE__, __LINE__); maybe_error(close(dstfd), "close", dstname, __FILE__, __LINE__); FREE(buf); surrounding_item = todo_item_destroy(surrounding_item); FREE(srcname); FREE(dstname); } static void do_dir_contents_item(struct todo_list *tl, const struct todo_reg_or_dir *item, struct todo_item *surrounding_item) // Effect: Copy the contents of src to dst, both of which be existing // directories. { assert(&todo_item_magic == surrounding_item->magic); assert(TODO_DIR_CONTENTS == surrounding_item->stage); assert(&surrounding_item->u.reg_or_dir == item); size_t n_dirents = 0; size_t dirents_limit = 2; struct dirent *MALLOC_N(dirents, dirents_limit); char *srcname = pathcatn(item->src_fprefix, item->fsuffix); char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); { DIR *d = opendir(srcname); if (d == NULL) fprintf(stderr, "Couldn't open %s\n", srcname); assert(d); struct dirent *entry; while ((entry = readdir(d))) { if (matches_an_exclude(item->fsuffix, entry->d_name) || !matches_an_include(item->fsuffix, entry->d_name)) continue; if (strcmp(entry->d_name, ".") == 0) continue; if (strcmp(entry->d_name, "..") == 0) continue; if (strcmp(entry->d_name, ".snapshot") == 0) continue; if (n_dirents >= dirents_limit) { dirents_limit*=2; REALLOC(dirents, dirents_limit); } dirents[n_dirents++] = *entry; } closedir(d); } if (pargs.delete) { //qsort(dirents, n_dirents, sizeof(*dirents), dirent_sorter); DIR *d = opendir(dstname); if (!d) { fprintf(stderr, "%s:%d Could not opendir(\"%s\")\n", __FILE__, __LINE__, dstname); } assert(d); struct dirent *entry; while ((entry = readdir(d))) { char *new_suffix = pathcatn(item->fsuffix, entry->d_name); if (1 && !matches_an_exclude(new_suffix, entry->d_name) && matches_an_include(new_suffix, entry->d_name) && strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0 && strcmp(entry->d_name, ".snapshot") != 0) { for (size_t i = 0; i < n_dirents; i++) { if (strcmp(entry->d_name, dirents[i].d_name) == 0) { FREE(new_suffix); goto keep_it; } } todo_list_push(tl, mk_todo_item_rm( item->dst_fprefix, new_suffix, strdup(entry->d_name), NULL)); } else { FREE(new_suffix); } keep_it: ;/* nothing */ } closedir(d); } for (size_t i = 0; i < n_dirents; i++) { char *new_suffix = pathcatn(item->fsuffix, dirents[i].d_name); // Ensure the existence immediately. If needed, this will create an // additional task to populate the item. if (!matches_an_exclude(new_suffix, dirents[i].d_name) && matches_an_include(new_suffix, dirents[i].d_name)) { do_item(tl, mk_todo_item_ensure_existence_and_populate( item->src_fprefix, item->dst_fprefix, new_suffix)); } else { FREE(new_suffix); } } // now set the permissions on the dir. struct stat sbuf; MAYBE_ERROR(lstat, srcname, &sbuf); int chown_ret = chown(dstname, sbuf.st_uid, sbuf.st_gid); maybe_error(chown_ret, "chown", dstname, __FILE__, __LINE__); int chmod_ret = chmod(dstname, sbuf.st_mode & (0777 | S_ISUID | S_ISGID)); maybe_error(chmod_ret, "chmod", dstname, __FILE__, __LINE__); struct timespec times[2] = {sbuf.st_atim, sbuf.st_mtim}; int utimensat_ret = utimensat(AT_FDCWD, dstname, times, 0); maybe_error(utimensat_ret, "utimensat", dstname, __FILE__, __LINE__); FREE(dirents); todo_item_destroy(surrounding_item); FREE(srcname); FREE(dstname); } static void do_ensure_dir(struct todo_list *tl, const struct todo_ensure *item, const struct stat *sbuf, bool dexists, const struct stat *dbuf); static void do_ensure_symlink(struct todo_list *tl, const struct todo_ensure *item, const struct stat *sbuf, bool dexists, const struct stat *dbuf); static void do_ensure_nod(struct todo_list *tl, const struct todo_ensure *item, const struct stat *sbuf, bool dexists, struct stat *dbuf); void do_create_item_obj(struct todo_list *tl, struct todo_item *item, bool dexists); static void do_ensure_existence_and_populate_item( struct todo_list *tl, const struct todo_ensure *item, struct todo_item *surrounding_item) // Effect: Ensure that the dst is created, and then if it is a directory or // regular file enqueue an item to copy the contents. { assert(&todo_item_magic == surrounding_item->magic); assert(TODO_ENSURE_EXISTENCE_AND_POPULATE == surrounding_item->stage); assert(&surrounding_item->u.ensure == item); struct stat sbuf; char *srcname = pathcatn(item->src_fprefix, item->fsuffix); char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); int er = lstat(srcname, &sbuf); if ( er != 0 && errno == ENOENT) { fprintf(stderr, "%s:%d Warning: Could not lstat(\"%s\"). errno=%d (%s)\n", __FILE__, __LINE__, srcname, errno, strerror(errno)); surrounding_item = todo_item_destroy(surrounding_item); FREE(srcname); FREE(dstname); return; } bool dexists; struct stat dbuf; { int r = lstat(dstname, &dbuf); if (r != 0 && errno == ENOENT) { dexists = false; } else if (r == 0) { dexists = true; } else { maybe_error(r, "lstat", dstname, __FILE__, __LINE__); assert(0); // unreachable } } if (S_ISDIR(sbuf.st_mode)) { do_ensure_dir(tl, item, &sbuf, dexists, &dbuf); } else if (S_ISLNK(sbuf.st_mode)) { do_ensure_symlink(tl, item, &sbuf, dexists, &dbuf); } else if (0 || S_ISREG(sbuf.st_mode) || S_ISCHR(sbuf.st_mode) || S_ISBLK(sbuf.st_mode) || S_ISFIFO(sbuf.st_mode) || S_ISSOCK(sbuf.st_mode)) { do_ensure_nod(tl, item, &sbuf, dexists, &dbuf); } else { assert(0); // unreachable } surrounding_item = todo_item_destroy(surrounding_item); FREE(srcname); FREE(dstname); } static void do_ensure_dir(struct todo_list *tl, const struct todo_ensure *item, const struct stat *sbuf, bool dexists, const struct stat *dbuf) { char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); if (!dexists) { // Dest doesn't exist. Create it and copy contents (which will // also fix the permissions of the directory). mkdir(dstname, 0770); } else if (S_ISDIR(dbuf->st_mode)) { // Dest exists as a dir. Make it writeable, copy contents (and // fix permissions). MAYBE_ERROR(chmod, dstname, 0770); } else { // Dest exists, but it's not a directory. Get rid of it, make the // dir, and copy contents. MAYBE_ERROR(unlink, dstname); mkdir(dstname, 0770); } // We could try to reuse the item, but that's not very clean. todo_list_push(tl, mk_todo_item_dir_contents(item->src_fprefix, item->dst_fprefix, strdupn(item->fsuffix), sbuf, dbuf)); FREE(dstname); } static void unlink_and_symlink(const struct todo_ensure *item, const char *target) { char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); MAYBE_ERROR(unlink, dstname); MAYBE_ERROR2(symlink, target, dstname); FREE(dstname); } static void do_ensure_symlink(struct todo_list *tl __attribute__((unused)), const struct todo_ensure *item, const struct stat *sbuf, bool dexists, const struct stat *dbuf) { char *srcname = pathcatn(item->src_fprefix, item->fsuffix); char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); size_t ssize = (size_t)sbuf->st_size; char *slinkval = malloc(ssize + 1); ssize_t sgot = readlink(srcname, slinkval, ssize+1); maybe_error(sgot < 0 || (size_t)sgot != ssize, "readlink", srcname, __FILE__, __LINE__); slinkval[ssize]=0; if (!dexists) { MAYBE_ERROR2(symlink, slinkval, dstname); } else if (S_ISDIR(dbuf->st_mode)) { delete_dir_recursively(dstname); MAYBE_ERROR2(symlink, slinkval, dstname); } else if (S_ISLNK(dbuf->st_mode)) { if (dbuf->st_size != sbuf->st_size) { unlink_and_symlink(item, slinkval); } else { size_t size = (size_t)dbuf->st_size; char *dlinkval = malloc(size + 1); ssize_t dgot = readlink(dstname, dlinkval, size+1); maybe_error(dgot < 0 || (size_t)dgot != size, "readlink", dstname, __FILE__, __LINE__); int cmp = memcmp(dlinkval, slinkval, size); FREE(dlinkval); if (cmp != 0) { unlink_and_symlink(item, slinkval); } else { // The existing symlink is what we want. /*nothing*/; } } } else { unlink_and_symlink(item, slinkval); } FREE(srcname); FREE(dstname); FREE(slinkval); } static void do_ensure_nod(struct todo_list *tl, const struct todo_ensure *item, const struct stat *sbuf, bool dexists, struct stat *dbuf) // Effect: Make sure that a (nondir) file object exists. If it's a regular // file then add a todo item to populate it. { char *srcname = pathcatn(item->src_fprefix, item->fsuffix); char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); // Here we have to do some extra work to make the topology of the inodes // match in the source and destionation. The first question is whether // the source inode has a corresponding destination inode already. lock_itopology_ownership(sbuf->st_dev, sbuf->st_ino); struct src_inode_value const * siv; bool already_copied = itopology_lookup_src(sbuf->st_dev, sbuf->st_ino, &siv); // The om is still held, so no one else can copy the inode until we // resolve this. if (already_copied) { // The file has already been copied, so all we need is a link. It's // resolved so we can release om. unlock_itopology_ownership(sbuf->st_dev, sbuf->st_ino); if (dexists) { // The destination already exists. if (S_ISDIR(dbuf->st_mode)) { // The destination exists, but it's a dir. Must rmdir the // directory, then make a link to the object. MAYBE_ERROR(rmdir, dstname); goto dest_doesnt_exist; } else if (1 && siv->dev == dbuf->st_dev && siv->ino == dbuf->st_ino) { // The link exists to the right thing, so we are done. /*nothing*/ ; } else { // Something exists there, but it's the wrong thing. MAYBE_ERROR(unlink, dstname); goto dest_doesnt_exist; } } else { // The destination doesn't exist. dest_doesnt_exist: MAYBE_ERROR2(link, siv->fname, dstname); } } else { // The file has not been copied, so we need to make sure the // destination exists and isn't an inode that *has* been previously // mapped. if (dexists) { // The destination exists. if (S_ISDIR(dbuf->st_mode)) { // The destination exists, but it's a dir. Must rmdir the // directory, then make a link to the object. MAYBE_ERROR(rmdir, dstname); goto destination_doesnt_exist; } else if (0 || ((dbuf->st_mode & S_IFMT) != (sbuf->st_mode & S_IFMT)) || itopology_lookup_dst(dbuf->st_dev, dbuf->st_ino)) { // It's a different file type or the dev and inode already has // been mapped, so we cannot use the current destination. MAYBE_ERROR(unlink, dstname); goto destination_doesnt_exist; } else { // dest exists, and we can reuse it. // Todo: If we don't have write permission, make it writeable. /*nothing*/ ; } } else { // The destination doesn't exist. destination_doesnt_exist: MAYBE_ERROR(mknod, dstname, sbuf->st_mode | 0700, sbuf->st_dev); MAYBE_ERROR(lstat, dstname, dbuf); } // Now we have the right destination inode. itopology_set(sbuf->st_dev, sbuf->st_ino, dbuf->st_dev, dbuf->st_ino, dstname); if (S_ISREG(sbuf->st_mode) && (0 || sbuf->st_size != dbuf->st_size || sbuf->st_mtim.tv_sec != dbuf->st_mtim.tv_sec || sbuf->st_mtim.tv_nsec != dbuf->st_mtim.tv_nsec)) { todo_list_push(tl, mk_todo_item_reg_contents(item->src_fprefix, item->dst_fprefix, strdupn(item->fsuffix), sbuf, dbuf)); } unlock_itopology_ownership(sbuf->st_dev, sbuf->st_ino); } FREE(srcname); FREE(dstname); } static void do_cp(const char *src, const char *dst) { struct todo_item *item = mk_todo_item_ensure_existence_and_populate( src, dst, NULL); struct todo_list *tl = mk_todo_list(); todo_list_run(tl, pargs.n_threads, item, do_item); todo_list_destroy(tl); } void help(int exitcode) { FILE *out = exitcode ? stderr : stdout; fprintf(out, "Usage: %s [OPTION] source dest\n", pargs.cmd); fprintf(out, "Recursive copy of a directory hierarchy, done in parallel for performance.\n"); fprintf(out, " OPTION can be\n"); fprintf(out, " -P P (capital P) use P-fold parallelism (default %d threads).\n", default_n_threads); fprintf(out, " --restore Recurse, put stop at file system boundaries, preserve hard links, symlinks permissions, modification times, group, owners, and special files such as named sockets and fifos. Similar to `-arxH` in rsync. Note: This does not include --delete.\n"); fprintf(out, " --delete Delete files that appear in dst but not src.\n"); fprintf(out, " --exclude-from=FILE FILE contains a list of glob(3)-style patterns, which will not be copied.\n"); fprintf(out, " --include-from=FILE FILE contains a list of glob(3)-style patterns, which will be copied. Everything else will be omitted.\n"); fprintf(out, " -- remaining arguments are files even if they start with '-'.\n"); fprintf(out, " -h, --help print help.\n"); fprintf(out, " --version print program version\n"); fprintf(out, " --history print a brief history of the versions\n"); fprintf(out, "This is version %s gitrev %s compiled %s %s.\n", VERSION, gitrev, __DATE__, __TIME__); fprintf(out, " Report bugs to parallel-tools-support_ww_grp@oracle.com\n"); exit(exitcode); } static int prefix_matches(const char *prefix, const char *string, char const **suffix) { while (*prefix == *string) { prefix++; string++; } if (*prefix == 0) { *suffix = string; return 1; } else { return 0; } } static void parse_args(int argc, const char *argv[], struct vector_of_strings *args) { assert(argc>0); pargs.cmd = argv[0]; argc--; argv++; while (argc) { char const *suffix; if (strcmp(argv[0], "-P") == 0) { argc--; argv++; if (argc == 0) help(1); pargs.n_threads = parse_number_or_help(argv[0]); } else if (strcmp(argv[0], "--delete") == 0) { pargs.delete = 1; } else if (strcmp(argv[0], "--restore") == 0) { pargs.ignore_times = 1; pargs.one_file_system = 1; pargs.hard_links = 1; pargs.preserve_symlinks = 1; pargs.preserve_perms = 1; pargs.preserve_times = 1; pargs.preserve_owner = 1; pargs.preserve_group = 1; pargs.preserve_devices = 1; pargs.preserve_specials = 1; } else if (prefix_matches("--exclude-from=", argv[0], &suffix)) { if (vector_of_strings_size(pargs.include_from)) { fprintf(stderr, "%s\n", "Options --exclude-from and --include-from are mutually exclusive."); help(1); } vector_of_strings_push(pargs.exclude_from, suffix); } else if (prefix_matches("--include-from=", argv[0], &suffix)) { if (vector_of_strings_size(pargs.exclude_from)) { fprintf(stderr, "%s\n", "Options --exclude-from and --include-from are mutually exclusive."); help(1); } vector_of_strings_push(pargs.include_from, suffix); } else if (strcmp(argv[0], "-h") == 0 || strcmp(argv[0], "--help") == 0) { help(0); } else if (strcmp(argv[0], "--version") == 0) { printf("%s %s\n" "Copyright (C) 2019 Oracle.\n" "Written by Bradley C. Kuszmaul\n", pargs.cmd, VERSION); exit(0); } else if (strcmp(argv[0], "--history") == 0) { fprintf(stderr, "%s", VERSION_HISTORY); exit(0); } else if (strcmp(argv[0], "--") == 0) { while (1) { argc--; argv++; vector_of_strings_push(args, argv[0]); if (argc==1) break; } } else { vector_of_strings_push(args, argv[0]); } argc--; argv++; } if (vector_of_strings_size(args) != 2) { fprintf(stderr, "Need two arguments for copying, got %zu\n", vector_of_strings_size(args)); help(1); } { char *line = NULL; size_t len = 0; ssize_t nread = 0; for (size_t i = 0; i < vector_of_strings_size(pargs.exclude_from); i++) { char const *fname = vector_of_strings_fetch(pargs.exclude_from, i); FILE *f = fopen(fname, "r"); if (f == NULL) { fprintf(stderr, "Could not open file named %s, " "mentioned as an --exclude-from\n", fname); FREE(line); exit(1); } while ((nread = getline(&line, &len, f)) != -1) { size_t llen = strlen(line); if (llen == 0) continue; if (';' == line[0]) continue; if ('#' == line[0]) continue; if (line[llen-1] == '\n') line[llen-1]=0; vector_of_strings_push(pargs.exclude, line); } fclose(f); } for (size_t i = 0; i < vector_of_strings_size(pargs.include_from); i++) { char const *fname = vector_of_strings_fetch(pargs.include_from, i); FILE *f = fopen(fname, "r"); if (f == NULL) { fprintf(stderr, "Could not open file named %s, " "mentioned as an --include-from\n", fname); FREE(line); exit(1); } while ((nread = getline(&line, &len, f)) != -1) { size_t llen = strlen(line); if (llen == 0) continue; if (';' == line[0]) continue; if ('#' == line[0]) continue; if (line[llen-1] == '\n') line[llen-1]=0; vector_of_strings_push(pargs.include, line); } fclose(f); } FREE(line); } } static void on_exit_destroy_parcp(int status __attribute__((unused)), void *ignore __attribute__((unused))) { vector_of_strings_destroy(pargs.exclude_from); vector_of_strings_destroy(pargs.exclude); vector_of_strings_destroy(pargs.include_from); vector_of_strings_destroy(pargs.include); vector_of_strings_destroy(pargs.vs); deinit_itopology(); } int main (int argc, const char *argv[]) { pargs.vs = mk_vector_of_strings(); pargs.exclude_from = mk_vector_of_strings(); pargs.exclude = mk_vector_of_strings(); pargs.include_from = mk_vector_of_strings(); pargs.include = mk_vector_of_strings(); init_itopology(); on_exit(on_exit_destroy_parcp, NULL); parse_args(argc, argv, pargs.vs); assert(vector_of_strings_size(pargs.vs) == 2); char *dst = vector_of_strings_pop(pargs.vs); char *src = vector_of_strings_pop(pargs.vs); struct stat statb; { int r = lstat(src, &statb); if (r != 0) { fprintf(stderr, "Could not lstat(\"%s\") src. errno=%d (%s)\n", src, errno, strerror(errno)); FREE(dst); FREE(src); exit(1); } if (!S_ISDIR(statb.st_mode)) { fprintf(stderr, "%s is not a directory.\n", src); fprintf(stderr, "parcp operates only on directories. To copy a single file, use cp\n"); FREE(dst); FREE(src); exit(1); } } { int r = mkdir(dst, S_IRUSR|S_IWUSR|S_IXUSR); if (r != 0 && errno != EEXIST) { fprintf(stderr, "Could not mkdir(\"%s\") dst. errno=%d (%s)\n", dst, errno, strerror(errno)); FREE(dst); FREE(src); exit(1); } } // Creating a copy of src to pass to basename function // as it may modify the input. char *src_copy = strdup(src); // Using POSIX version of basename routine(picked from '') // instead of GNU verison(picked from ''). // Including '' either before or after including '', // will override the GNU variant of basename(). char *src_basename = basename(src_copy); if (src_basename == NULL) { fprintf(stderr,"Could not extract basename of src \"%s\"", src_basename); FREE(dst); FREE(src); exit(1); } char *old_dest = dst; dst = pathcatn(old_dest, src_basename); free(old_dest); do_cp(src, dst); free(dst); free(src); free(src_copy); return 0; }