commit 7e8045d83e4bef8685de6a2d7b550b499afcc103 Author: David Sorber Date: Sun Jul 16 10:08:47 2023 -0400 Adding Orcale's fss-parallel-tools taken from fss-parallel-tools-1.58-1.el8.src.rpm. diff --git a/software/fss-parallel-tools/LICENSE b/software/fss-parallel-tools/LICENSE new file mode 100644 index 0000000..609fb03 --- /dev/null +++ b/software/fss-parallel-tools/LICENSE @@ -0,0 +1,36 @@ +Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved. + +The Universal Permissive License (UPL), Version 1.0 + +Subject to the condition set forth below, permission is hereby granted to any +person obtaining a copy of this software, associated documentation and/or data +(collectively the "Software"), free of charge and under any and all copyright +rights in the Software, and any and all patent rights owned or freely licensable +by each licensor hereunder covering either (i) the unmodified Software as +contributed to or provided by such licensor, or (ii) the Larger Works (as +defined below), to deal in both + +(a) the Software, and +(b) any piece of software and/or hardware listed in the lrgrwrks.txt file if one +is included with the Software (each a “Larger Work” to which the Software is +contributed by such licensors), + +without restriction, including without limitation the rights to copy, create +derivative works of, display, perform, and distribute the Software and make, +use, sell, offer for sale, import, export, have made, and have sold the Software +and the Larger Work(s), and to sublicense the foregoing rights on either these +or other terms. + +This license is subject to the following condition: + +The above copyright notice and either this complete permission notice or at a +minimum a reference to the UPL must be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/software/fss-parallel-tools/Makefile b/software/fss-parallel-tools/Makefile new file mode 100644 index 0000000..da4860e --- /dev/null +++ b/software/fss-parallel-tools/Makefile @@ -0,0 +1,636 @@ +## +# Copyright © 2019, Oracle and/or its affiliates. All rights reserved. +# +# Licensed under the Universal Permissive License v 1.0 as shown at +# http://oss.oracle.com/licenses/upl. +# +# Update this (and the VERSION_HISTORY in version.h) when the version changes. +VERSION = 1.58 + +#SAN = -fsanitize=thread +#SAN = -fsanitize=address +#GCOV_FLAGS = -fprofile-arcs -ftest-coverage +CFLAGS=-Wall -Wextra -Wshadow -Wconversion -Wpointer-arith -Wmissing-prototypes -Wwrite-strings -Wswitch-enum -Werror -g -O1 -pthread -std=gnu99 $(SAN) $(GCOV_FLAGS) +CPPFLAGS=-DVERSION=\"$(VERSION)\" +#CPPFLAGS+=-DSPEW_ENABLED + +# It turns out that we need -lrt (probably because of clock_gettime() +# or else we'll end up with a dependency on glibc 2.17, which often is +# too advanced (e.g., on Oracle Linux 6.9) +LDLIBS=-lrt +LDFLAGS=-pthread $(SAN) $(GCOV_FLAGS) + +VALGRIND = valgrind --leak-check=full --error-exitcode=3 -q + +PARTAR = partar + +default: $(PARTAR) parcp parrm $(PARTAR).1.gz parcp.1.gz parrm.1.gz + +# Don't delete intermediate files (specify .SECONDARY with no dependencies) +.SECONDARY: + +rpms: ../fss-parallel-tools-$(VERSION).tar.gz fss-parallel-tools.spec + mkdir -p ~/rpmbuild/SOURCES + mkdir -p ~/rpmbuild/SPECS + cp ../fss-parallel-tools-$(VERSION).tar.gz ~/rpmbuild/SOURCES + cp fss-parallel-tools.spec ~/rpmbuild/SPECS + cd ~/rpmbuild/SPECS;rpmbuild -ba fss-parallel-tools.spec +fss-parallel-tools.spec: fss-parallel-tools.spec.in Makefile + sed s/VERSION_DEFINED_IN_MAKEFILE/$(VERSION)/ < $< > $@ +mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST))) +current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path)))) +SOURCES = gitrev.c $(wildcard *.c *.h common/*.c common/*.h) Makefile LICENSE + +../fss-parallel-tools-$(VERSION).tar.gz: $(SOURCES) + tar czf $@ --transform 's,^,fss-parallel-tools/,' $(SOURCES) +tarball: ../fss-parallel-tools-$(VERSION).tar.gz + +todo.o: todo.h +dircache.o: dircache.h malloc.h +htable.o: htable.h malloc.h +fwg.o: malloc.h fwg.h hashfun.h +stringset.o: stringset.h hashfun.h malloc.h + +#################### +# COMMON +#################### +# sources +common/assert.o: common/assert.h common/backtrace.h common/spew.h version.h +COMMON_O += common/assert.o +COMMON_O += common/backtrace.o +common/backtrace.o: common/spew.h common/simplest-defs.h common/sparse.h common/backtrace.h +COMMON_O += common/debug-tag.o +common/debug-tag.o: common/debug-tag.h common/simplest-defs.h common/sparse.h +COMMON_O += common/error.o +common/error.o: common/error.h common/simplest-defs.h common/sparse.h +COMMON_O += common/spew.o +common/spew.o: common/spew.h common/backtrace.h common/simplest-defs.h common/tsan.h version.h common/debug-tag.h common/sparse.h common/error.h + +fdleak.o: fdleak.h +putils.o: putils.h common/spew.h +parcp.o: putils.h malloc.h version.h sched.h vector-of-strings.h common/spew.h +$(PARTAR).o: putils.h malloc.h version.h sched.h vector-of-strings.h inodemap.h stringset.h vec.h fdleak.h fwg.h common/spew.h +parrm.o: putils.h malloc.h version.h sched.h vector-of-strings.h common/spew.h +vector-of-strings.o: vector-of-strings.h malloc.h +$(PARTAR): $(PARTAR).o putils.o sched.o vector-of-strings.o gitrev.o fwg.o stringset.o vec.o fdleak.o inodemap.o $(COMMON_O) +inodemap.o: inodemap.h malloc.h +parcp: parcp.o putils.o sched.o vector-of-strings.o gitrev.o inodemap.o todo.o $(COMMON_O) +parrm: parrm.o putils.o gitrev.o sched.o vector-of-strings.o $(COMMON_O) + +.DELETE_ON_ERROR: + +.SUFFIX: .1.gz +%.1.gz: % + PATH=$$PATH:. help2man -N --name="parallel implementation of $(subst par,,$<)" $< | gzip - > $@ + +gitrev.o: gitrev.c +# Don't create a new gitrev.c file if nothing changed. We don't want +# to rerun all the tests just because I modified the Makefile. +gitrev.c: $(filter-out gitrev.c,$(wildcard *.c *.h common/*.c common/*.h Makefile)) + echo \#include \"version.h\" > gitrev_new.c + echo const char gitrev[] = \"$$(git diff-index --quiet HEAD -- || echo "++")$$(git rev-parse HEAD)\"\; >> gitrev_new.c + if ! diff gitrev_new.c gitrev.c >> /dev/null ; then mv gitrev_new.c gitrev.c; fi +setup: $(PARTAR) + rm -rf tar-1.30 + +# Tests + +.PHONY: check +TESTS = fifo tar-1.30 linux-4.15 longfilename3 +TESTS_T = $(patsubst %, %.test, $(TESTS)) +check: parrm_eaccess_test.test +check: parrm_eaccess_test2.test +check: parrm_eaccess_test3.test +check: parrm_eaccess_test4.test +check: excld.test +check: excld-equal.test +check: error.test +check: linklink.test +check: permissions.test +check: $(TESTS_T) test_fib.run +check: decompression-errors.test +check: parcp0.test parcp1.test parcp2.test parcp3.test parcp4.test parcp5.test parcp6.test parcp7.test +.SUFFIX: .test +%.test: %.partar_test %.parcp_test %.parrm_test %.tar_test + touch $@ + +.SUFFIX: .partar_test +%.partar_test: %.partar1_diff %.partar2_diff %.partar3_diff %.partar4_diff partar1.test + touch $@ + +test_fib.run: test_fib + ./test_fib + touch $@ + +$(info VAR="$(COMMON_O)") + +test_fib: test_fib.o sched.o $(COMMON_O) gitrev.o +test_fib.o: malloc.h sched.h common/spew.h +sched.o: malloc.h sched.h common/spew.h + +# + +# These tests run partar in various ways and compare the output to what happened with tar. +# partar_diff1_test runs "partar pxf %.tar" +# partar_diff2_test runs "partar pxf - < %.tar" +# partar_diff3_test runs "partar pxzf %.tar.gz" +# partar_diff4_test runs "partar pxzf - < %.tar.gz" + +# partar_test has things that are fifos in it, and we want to ignore +# diff error messages about those. That's what the grep is about in +# the diff line. + +output-with-tar output-with-partar1 output-with-partar2 output-with-partar3 output-with-partar4 uncompressed-tars gzipped-tars: + mkdir -p $@ + +uncompressed-tars/%.tar: test-tars/%.tar.xz | uncompressed-tars + xzcat $< > $@ + +gzipped-tars/%.tar.gz: uncompressed-tars/%.tar | gzipped-tars + gzip < $< > $@ + +%.tar_done: uncompressed-tars/%.tar | output-with-tar + rm -rf $(patsubst uncompressed-tars/%.tar,output-with-tar/%, $<) + tar xf $< -C output-with-tar + touch $@ + +# partar1 tests makes sure that partar works with a named file +%.partar1_done: uncompressed-tars/%.tar $(PARTAR) | output-with-partar1 + rm -rf $(patsubst uncompressed-tars/%.tar,output-with-partar1/%, $<) + $(VALGRIND) ./$(PARTAR) xf $< -C output-with-partar1 + touch $@ + +%.partar1_diff: %.tar_done %.partar1_done + diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar1/%,$<) | (! grep -v "is a fifo.*is a fifo") + touch $@ + +# partar2 tests make sure that partar works from stdin +%.partar2_done: uncompressed-tars/%.tar $(PARTAR) | output-with-partar2 + rm -rf $(patsubst uncompressed-tars/%.tar,output-with-partar2/%, $<) + $(VALGRIND) ./$(PARTAR) xf - -C output-with-partar2 < $< + touch $@ + +%.partar2_diff: %.tar_done %.partar2_done + diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar2/%,$<) | (! grep -v "is a fifo.*is a fifo") + touch $@ + +# partar3 tests make sure that partar works with a gzipped named file. +.SUFFIX: .partar3_done +%.partar3_done: gzipped-tars/%.tar.gz $(PARTAR) | output-with-partar3 + rm -rf $(patsubst gzipped-tars/%.tar.gz,output-with-partar3/%, $<) + $(VALGRIND) ./$(PARTAR) xzf $< -C output-with-partar3 + touch $@ + +.SUFFIX: .partar3_diff +%.partar3_diff: %.tar_done %.partar3_done + diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar3/%,$<) | (! grep -v "is a fifo.*is a fifo") + touch $@ + +# partar4 tests make sure that partar works with a gzipped stdin +%.partar4_done: gzipped-tars/%.tar.gz $(PARTAR) | output-with-partar4 + rm -rf $(patsubst gzipped-tars/%.tar.gz,output-with-partar4/%, $<) + $(VALGRIND) ./$(PARTAR) xzf - -C output-with-partar4 < $< + touch $@ + +%.partar4_diff: %.tar_done %.partar4_done + diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar4/%,$<) | (! grep -v "is a fifo.*is a fifo") + touch $@ + +# Test untarring onto a read-only file +partar1.test: $(PARTAR) | partar1.test.dir + rm -rf $@.dir/src $@.dir/dst + mkdir -p $@.dir/src $@.dir/dst + touch $@.dir/dst/readonly + chmod guo-w $@.dir/dst/readonly + echo contents > $@.dir/src/readonly + chmod guo-w $@.dir/src/readonly + set -e; set -o pipefail; $(VALGRIND) ./$(PARTAR) c -C $@.dir/src readonly|$(VALGRIND) ./$(PARTAR) x -C $@.dir/dst + diff -r $@.dir/{src,dst} + touch $@ + +output-with-parcp output-to-rm output-of-partarred: + mkdir -p $@ + +.SUFFIX: .parcp_test +# Turn off valgrind for the big linux parcp +linux-4.15.parcp_test: VALGRIND= +%.parcp_test: parcp %.tar_done | output-with-parcp + rm -rf $(patsubst %.parcp_test,output-with-parcp/%, $@) + $(VALGRIND) ./parcp -P 3 $(patsubst %.parcp_test,output-with-tar/%, $@) $(patsubst %.parcp_test,output-with-parcp/%, $@) + diff --no-dereference -r $(patsubst %.parcp_test,output-with-tar/%, $@) $(patsubst %.parcp_test,output-with-parcp/%, $@) | (! grep -v "is a fifo.*is a fifo") + touch $@ + +.SUFFIX: .parrm_test +%.parrm_test: uncompressed-tars/%.tar parrm $(PARTAR) | output-to-rm + $(VALGRIND) ./$(PARTAR) xf - -C output-to-rm < $< + test -d $(patsubst %.parrm_test,output-to-rm/%,$@) + $(VALGRIND) ./parrm -P 3 $(patsubst %.parrm_test,output-to-rm/%,$@) + test ! -d $(patsubst %.parrm_test,output-to-rm/%,$@) + touch $@ + +# This test only works on a few cases, since the order of the files isn't unique +.SUFFIX: .tardiff_test +%.tardiff_test: %.tar_out %.partar_out + diff $^ + touch $@ + +.SUFFIX: .tar_test +%.tar_test: %.partar_out %.tar_done | output-of-partarred + rm -rf $(patsubst %.partar_out,output-of-partarred/%,$<) + $(VALGRIND) ./$(PARTAR) xf $< -C output-of-partarred + diff --no-dereference -r $(patsubst %.tar_test,output-with-tar/%,$@) $(patsubst %.tar_test,output-of-partarred/%,$@) | (! grep -v "is a fifo.*is a fifo") + $(EXTRA_TEST) + +# These two tests check to see what happens if parrm runs into a +# directory that it hasn't got permission to access. The whole +# directory, or a subdirectory, respectively. + +ED = parrm_eaccess_test_dir +parrm_eaccess_test.test: parrm + -chmod u+rwx $(ED) > /dev/null 2>&1 ;true + rm -rf $(ED) + mkdir $(ED) + touch $(ED)/foo + chmod ugo-rwx $(ED) + rm -rf $(ED) > parrm_eaccess_test.out 2>&1 ; test $$? = 1 + fgrep Permission parrm_eaccess_test.out > /dev/null + ./parrm $(ED) > parrm_eaccess_test.out 2>&1 ; test $$? = 1 + fgrep Permission parrm_eaccess_test.out > /dev/null + touch $@ + +ED2 = parrm_eaccess_test2_dir +parrm_eaccess_test2.test: parrm + chmod -R u+rwx $(ED2) > /dev/null 2>&1 ;true + rm -rf $(ED2) + mkdir -p $(ED2)/$(ED2) + touch $(ED2)/$(ED2)/foo + touch $(ED2)/foo + chmod ugo-rwx $(ED2)/$(ED2) + ./parrm $(ED2) > $(ED2).out 2>&1 ; test $$? = 1 + chmod -R u+rwx $(ED2) > /dev/null 2>&1 ;true + test ! -f $(ED2)/foo + test -f $(ED2)/$(ED2)/foo + touch $@ + +# And these are the same tests where we can access the dir but we cannot unlink it. +ED3 = parrm_eaccess_test3_dir +parrm_eaccess_test3.test: parrm + -chmod u+rwx $(ED3) > /dev/null 2>&1 ;true + rm -rf $(ED3) + mkdir $(ED3) + touch $(ED3)/foo + chmod ugo-w $(ED3) + ls -la $(ED3) + rm -rf $(ED3) > $(ED3).rm.out 2>&1 ; test $$? = 1 + fgrep Permission $(ED3).rm.out > /dev/null + ./parrm $(ED3) > $(ED3).parrm.out 2>&1 ; test $$? = 1 + fgrep Permission $(ED3).parrm.out > /dev/null + touch $@ + +ED4 = parrm_eaccess_test4_dir +parrm_eaccess_test4.test: parrm + chmod -R u+rwx $(ED4) > /dev/null 2>&1 ;true + rm -rf $(ED4) + mkdir -p $(ED4)/$(ED4) + touch $(ED4)/$(ED4)/foo + touch $(ED4)/foo + chmod ugo-rx $(ED4)/$(ED4) + ./parrm $(ED4) > $(ED4).out 2>&1 ; test $$? = 1 + chmod -R u+rwx $(ED4) > /dev/null 2>&1 ;true + test ! -f $(ED4)/foo + test -f $(ED4)/$(ED4)/foo + touch $@ + +# Some tarballs have extra requirements that are hard to capture with diff. We'd really like to verify that the topology of two directories is the same, but I don't know how to get diff to really do it. So we'll have an ad hoc test that these two files really are the same. +longfilename3.tar_test: EXTRA_TEST = test output-of-partarred/longfilename3/aaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbccccccccccccccccdddddddddddddddeeeeeeeeeeeeeeffffffffffffffg -ef output-of-partarred/longfilename3/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + +.SUFFIX: .tar_out .partar_out +%.tar_out: %.tar_done + tar cf $@ $(patsubst %.tar_done,output-with-tar/%,$<) +%.partar_out: %.tar_done $(PARTAR) + (cd output-with-tar;../$(PARTAR) cf - $(patsubst %.tar_done,%,$<)) > $@ + +error.test: $(PARTAR) + ! ./$(PARTAR) xf - < /dev/null 2> error.test.out + fgrep -q ": This does not look like a tar archive" error.test.out + fgrep -q ": Exiting with failure status due to previous errors" error.test.out + ! ./$(PARTAR) xzf - < /dev/null 2> error.test.out + fgrep -q "gzip: stdin: unexpected end of file" error.test.out + fgrep -q ": Child returned status 1" error.test.out + fgrep -q ": Exiting with failure status due to previous errors" error.test.out + ! ./$(PARTAR) cf error.tar output-with-tar/fifo no-such-file output-with-tar/fifo 2> error.test.out + fgrep -q ": no-such-file: Cannot stat: No such file or directory" error.test.out + fgrep -q ": Exiting with failure status due to previous errors" error.test.out + touch $@ + +# Test excluding files both in "c" mode and in "x" mode. +excld.test: tar-1.30.tar_done $(PARTAR) + rm -rf excld.dir.tar excld.dir.partar excld.odir.tar excld.odir.partar + mkdir excld.dir.tar excld.dir.partar excld.odir.tar excld.odir.partar + tar cf - -C output-with-tar/ --exclude d\?c tar-1.30 | tar xf - -C excld.dir.tar + $(VALGRIND) ./$(PARTAR) cf - -C output-with-tar/ --exclude d\?c tar-1.30 | tar xf - -C excld.dir.partar + diff -r excld.dir.tar excld.dir.partar + tar c -C output-with-tar/ tar-1.30 | tar x -C excld.odir.tar --exclude d\?c + diff -r excld.dir.tar excld.odir.tar + tar c -C output-with-tar/ tar-1.30 | $(VALGRIND) ./$(PARTAR) x -C excld.odir.partar --exclude d\?c + diff -r excld.dir.tar excld.odir.partar + touch $@ + +# Test excluding files both in "c" mode and in "x" mode same as above but uses --exclude=PATTERN. +excld-equal.test: tar-1.30.tar_done $(PARTAR) + rm -rf excld-equal.dir.tar excld-equal.dir.partar excld-equal.odir.tar excld-equal.odir.partar + mkdir excld-equal.dir.tar excld-equal.dir.partar excld-equal.odir.tar excld-equal.odir.partar + tar cf - -C output-with-tar/ --exclude=d\?c tar-1.30 | tar xf - -C excld-equal.dir.tar + $(VALGRIND) ./$(PARTAR) cf - -C output-with-tar/ --exclude=d\?c tar-1.30 | tar xf - -C excld-equal.dir.partar + diff -r excld-equal.dir.tar excld-equal.dir.partar + tar c -C output-with-tar/ tar-1.30 | tar x -C excld-equal.odir.tar --exclude d\?c + diff -r excld-equal.dir.tar excld-equal.odir.tar + tar c -C output-with-tar/ tar-1.30 | $(VALGRIND) ./$(PARTAR) x -C excld-equal.odir.partar --exclude=d\?c + diff -r excld-equal.dir.tar excld-equal.odir.partar + touch $@ + +# Test the same file appearing twice on the command line. tar makes a hardlink from the file to itself. +linklink.test: $(PARTAR) + echo a > linklink.txt + tar cf linklink.tar linklink.txt linklink.txt + $(VALGRIND) ./$(PARTAR) cf linklink.partar linklink.txt linklink.txt + # Don't run the diff unless we have a recent version of tar + if (tar --version|head -1|fgrep 1.3 > /dev/null); then diff linklink.tar linklink.partar; fi + mkdir -p linklink.tdir + tar xf linklink.tar -C linklink.tdir + mkdir -p linklink.pdir + tar xf linklink.partar -C linklink.pdir + diff -r linklink.tdir linklink.pdir + touch $@ + +# This isn't part of the standard tests, since it requires sudo to run properly. +permissions.sudo.test: $(PARTAR) + sudo rm -rf permissions.sudopartar.testdir permissions.sudountar.testdir + mkdir -p permissions.sudopartar.testdir + mkdir -p permissions.sudountar.testdir + sudo tar pxf test-tars/permissions.tar -C permissions.sudountar.testdir + sudo ./$(PARTAR) pxf test-tars/permissions.tar -C permissions.sudopartar.testdir + # for some reason tar doesn't set the mtime on partar/src/bin in those directories, so let's ignore bin. + (cd permissions.sudopartar.testdir; ls -lR)|sort | egrep -v "bin$$" > permissions.sudopartar.out + (cd permissions.sudountar.testdir; ls -lR)|sort | egrep -v "bin$$" > permissions.sudountar.out + diff permissions.sudopartar.out permissions.sudountar.out + +# permissions test for not sudo +permissions.test: $(PARTAR) + rm -rf permissions.in.dir permissions.tar.out.dir permissions.partar.out.dir permissions.tar.ls permissions.partar.ls + mkdir -p permissions.in.dir/s + mkdir -p permissions.in.dir/s/d1 + mkdir -p permissions.in.dir/s/d2 + ls -l permissions.in.dir/s + chmod 770 permissions.in.dir/s/d1 + chmod 750 permissions.in.dir/s/d2 + ls -l permissions.in.dir/s + mkdir -p permissions.tar.out.dir + tar cf - -C permissions.in.dir s | tar xf - -C permissions.tar.out.dir + ls -l permissions.tar.out.dir/s | sort > permissions.tar.ls + mkdir -p permissions.partar.out.dir + tar cf - -C permissions.in.dir s | $(VALGRIND) ./$(PARTAR) xf - -C permissions.partar.out.dir + ls -l permissions.partar.out.dir/s | sort > permissions.partar.ls + diff permissions.tar.ls permissions.partar.ls + touch $@ + +# Test for detecting .gz and .xz inputs. +decompression-errors.test: ./partar + ! ./partar xf test-tars/fifo.tar.xz 2> decompression-errors.out + fgrep "Input does not appear to be a tar file." decompression-errors.out > /dev/null + fgrep "Input appears to be xz compressed" decompression-errors.out > /dev/null + !(xzcat test-tars/fifo.tar.xz |gzip|./partar xf - 2> decompression-errors.out) + fgrep "Input does not appear to be a tar file." decompression-errors.out > /dev/null + fgrep "appears to be gzipped" decompression-errors.out > /dev/null + touch $@ + +%.dir: + mkdir $@ + +# Some specific tests for parcp +parcp0.test: parcp tar-1.30.tar_done | parcp0.test.dir + rm -rf $@.dir/* + ./parcp --restore -- output-with-tar/tar-1.30 $@.dir + touch $@ + +parcp1.test: parcp + ./parcp --version |fgrep $(VERSION) > /dev/null + ./parcp --help |fgrep OPTION > /dev/null + touch $@ + +# test that if we copy into an existing file that is longer, that it's properly trunated +parcp2.test: parcp | parcp2.test.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst/src + echo "hello" > $@.dir/src/hello + echo "hello there" > $@.dir/dst/src/hello + ls -i $@.dir/dst/src/hello > $@.dir/original.inode + ./parcp -P 3 $@.dir/src $@.dir/dst + ls -i $@.dir/dst/src/hello > $@.dir/new.inode + diff $@.dir/src/hello $@.dir/dst/src/hello + diff $@.dir/new.inode $@.dir/original.inode + touch $@ + +parcp3.test: parcp | parcp3.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst + # source and dest are the same symlink, so expect no change + ln -s unchanged $@.dir/src/uc + ln -s unchanged $@.dir/dst/uc + # source and dest are slightly different symlinks, the same length. + ln -s changed0 $@.dir/src/ch + ln -s changed1 $@.dir/dst/ch + # source and dest are different symlinks of different length. + ln -s shortch $@.dir/src/chlen + ln -s longerch $@.dir/dst/chlen + # source is symlink, dest is not there + ln -s notthere $@.dir/src/nt + # source is a symlink, dest is a dir + ln -s dest-was-dir $@.dir/src/wasdir + mkdir $@.dir/dst/wasdir + # source is a symlink, dest is a regular file. + ln -s dest-was-reg $@.dir/src/wasreg + touch $@.dir/dst/wasreg + #ls -l $@.dir/dst + ls -i parcp3.test.dir/dst/uc > $@.dir/original.uc.symlink + # source and dest are fifos. Make sure the inode is reused + mknod $@.dir/src/src_fifo_dst_fifo p + mknod $@.dir/dst/src_fifo_dst_fifo p + ls -i $@.dir/dst/src_fifo_dst_fifo > $@.dir/src_fifo_dst_fifo.original + # source is fifo and dest is regular. Make sure dest becomes fifo + mknod $@.dir/src/src_fifo_dst_reg p + touch $@.dir/dst/src_fifo_dst_reg + # source is regular and dest is fifo. Make sure dest becomes reg + touch $@.dir/src/src_reg_dst_fifo + mknod $@.dir/dst/src_reg_dst_fifo p + # source is file, dest is dir + touch $@.dir/src/src_reg_dst_dir + mkdir $@.dir/dst/src_reg_dst_dir + # source is dir, dest is file + mkdir $@.dir/src/src_dir_dst_reg + touch $@.dir/dst/src_dir_dst_reg + # two files that are the same inode in src, nothing in dst + touch $@.dir/src/src_two_inodes_dst_null.1 + ln $@.dir/src/src_two_inodes_dst_null.1 $@.dir/src/src_two_inodes_dst_null.2 + # two files the same inode in src, a file in dst + touch $@.dir/src/src_two_inodes_dst_reg.1 + ln $@.dir/src/src_two_inodes_dst_reg.1 $@.dir/src/src_two_inodes_dst_reg.2 + touch $@.dir/dst/src_two_inodes_dst_reg.1 + # two files the same inode in src, two dirs in dst + touch $@.dir/src/src_two_inodes_dst_two_dirs.1 + ln $@.dir/src/src_two_inodes_dst_two_dirs.1 $@.dir/src/src_two_inodes_dst_two_dirs.2 + mkdir $@.dir/dst/src_two_inodes_dst_two_dirs.1 + mkdir $@.dir/dst/src_two_inodes_dst_two_dirs.2 + # three files the same inode in src, three files in dst, two of which are the same inode. + touch $@.dir/src/src_3_inodes_dst_3_files.1 + ln $@.dir/src/src_3_inodes_dst_3_files.1 $@.dir/src/src_3_inodes_dst_3_files.2 + ln $@.dir/src/src_3_inodes_dst_3_files.1 $@.dir/src/src_3_inodes_dst_3_files.3 + touch $@.dir/dst/src_3_inodes_dst_3_files.1 + ln $@.dir/dst/src_3_inodes_dst_3_files.1 $@.dir/dst/src_3_inodes_dst_3_files.2 + touch $@.dir/dst/src_3_inodes_dst_3_files.3 + # do the copy + $(VALGRIND) ./parcp $@.dir/src $@.dir/dst + ls -l $@.dir/dst + test -h $@.dir/dst/ch + ls -l $@.dir/dst/ch | fgrep "ch -> changed0" + ls -l $@.dir/dst/nt | fgrep "nt -> notthere" + ls -l $@.dir/dst/wasdir | fgrep "wasdir -> dest-was-dir" + ls -l $@.dir/dst/chlen | fgrep "chlen -> shortch" + test -h $@.dir/dst/wasreg + ls -l $@.dir/dst/wasreg| fgrep "wasreg -> dest-was-reg" + # test follows symlinks so we must do htis more carefully + ls -i parcp3.test.dir/dst/uc > $@.dir/new.uc.symlink + diff $@.dir/original.uc.symlink $@.dir/new.uc.symlink + ls -i $@.dir/dst/src_fifo_dst_fifo > $@.dir/src_fifo_dst_fifo.new + diff $@.dir/src_fifo_dst_fifo.new $@.dir/src_fifo_dst_fifo.original + test -p $@.dir/dst/src_fifo_dst_reg + test -f $@.dir/dst/src_reg_dst_fifo + test -f $@.dir/dst/src_reg_dst_dir + test -d $@.dir/dst/src_dir_dst_reg + test parcp3.test.dir/dst/src_two_inodes_dst_null.1 -ef parcp3.test.dir/dst/src_two_inodes_dst_null.2 + test parcp3.test.dir/dst/src_two_inodes_dst_reg.1 -ef parcp3.test.dir/dst/src_two_inodes_dst_reg.2 + test parcp3.test.dir/dst/src_3_inodes_dst_3_files.1 -ef parcp3.test.dir/dst/src_3_inodes_dst_3_files.2 + test parcp3.test.dir/dst/src_3_inodes_dst_3_files.1 -ef parcp3.test.dir/dst/src_3_inodes_dst_3_files.3 + touch $@ + +# Check --delete +parcp4.test: parcp | parcp4.test.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst + mkdir -p $@.dir/dst/a/b/c/d/e + touch $@.dir/dst/a/b/c/d/e/f + mkdir -p $@.dir/dst/a/b/c/d/e2 + touch $@.dir/dst/a/b/c/d/e2/f + mkdir -p $@.dir/src/x/y/z + mkdir -p $@.dir/src/a/w + touch $@.dir/src/a/w/foo + $(VALGRIND) ./parcp --delete $@.dir/src $@.dir/dst + diff -r $@.dir/src $@.dir/dst + touch $@ + +# Check --exclude-from +parcp5.test: parcp | parcp5.test.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst + mkdir $@.dir/src/wanted + mkdir $@.dir/src/unwanted + touch $@.dir/src/wanted/file1.keep + touch $@.dir/src/wanted/file2.nokeep + touch $@.dir/src/wanted/file3.nokeep + echo unwanted > $@.dir/excludelist + echo wanted/file2.nokeep >> $@.dir/excludelist + echo file3.nokeep >> $@.dir/excludelist + $(VALGRIND) ./parcp --exclude-from=$@.dir/excludelist $@.dir/src $@.dir/dst + test -d $@.dir/dst/wanted + test -f $@.dir/dst/wanted/file1.keep + ls $@.dir/dst/wanted|fgrep -v file2.nokeep > /dev/null + ls $@.dir/dst/wanted|fgrep -v file3.nokeep > /dev/null + test ! -d $@.dir/dst/unwanted + touch $@ + +# Check --exclude-from with --delete +parcp6.test: parcp Makefile | parcp6.test.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst + mkdir -p $@.dir/src/src_wanted + mkdir -p $@.dir/dst/dst_wanted + mkdir -p $@.dir/dst/dst_extra + mkdir -p $@.dir/dst/dst_extra2 + mkdir -p $@.dir/dst/dst_kill + touch $@.dir/dst/dst_wanted/keepme + touch $@.dir/dst/dst_extra/dst_wanted_file + touch $@.dir/dst/dst_extra2/keep2 + touch $@.dir/dst/dst_extra2/killme + touch $@.dir/dst/dst_kill/killme2 + echo dst_wanted > $@.dir/excludelist + echo dst_wanted_file >> $@.dir/excludelist + echo dst_extra2/keep2 >> $@.dir/excludelist + $(VALGRIND) ./parcp --exclude-from=$@.dir/excludelist --delete $@.dir/src $@.dir/dst + test -f $@.dir/dst/dst_wanted/keepme + test -f $@.dir/dst/dst_extra/dst_wanted_file + test -f $@.dir/dst/dst_extra2/keep2 + test ! -f $@.dir/dst/dst_extra2/killme + test ! -f $@.dir/dst/dst_kill/killme2 + test ! -d $@.dir/dst/dst_kill + touch $@ + +# Check target contains a read-only file +# Three cases: 1) No such file in src +# 2) Exactly the same file in src +# 3) A different file in src +parcp7.test: parcp | parcp7.test.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst + touch $@.dir/dst/no-such-file-in-src + chmod ugo-w $@.dir/dst/no-such-file-in-src + echo same in src > $@.dir/src/same-file-in-src + chmod ugo-w $@.dir/src/same-file-in-src + cp -a $@.dir/src/same-file-in-src $@.dir/dst/same-file-in-src + echo diffrent in src > $@.dir/src/diff-file-in-src + chmod ugo-w $@.dir/src/diff-file-in-src + echo diffrent in dst dst > $@.dir/dst/diff-file-in-src + chmod ugo-w $@.dir/dst/diff-file-in-src + ls -l $@.dir/{src,dst} + $(VALGRIND) ./parcp --delete $@.dir/src $@.dir/dst + ls -l $@.dir/{src,dst} + test ! -f $@.dir/dst/no-such-file-in-src + test -f $@.dir/dst/diff-file-in-src + test -f $@.dir/dst/same-file-in-src + diff -r $@.dir/{src,dst} + touch $@ + +parcp8.test: parcp | parcp7.test.dir + rm -rf $@.dir/* + mkdir -p $@.dir/src $@.dir/dst + touch $@.dir/src/rws-file + chmod +sx $@.dir/src/rws-file + ./parcp $@.dir/src $@.dir/dst + test -x parcp8.test.dir/dst/rws-file + test -u parcp8.test.dir/dst/rws-file + echo touch $@ + +.PHONY: clean +clean: + rm -rf parrm parcp $(PARTAR) parrm.1.gz parcp.1.gz $(PARTAR).1.gz *.o output-with-tar output-to-rm output-with-parcp *.tar_done *.parrm_test *.parcp_test *.partar_test + rm -rf gitrev.c + rm -rf output-with-partar{1,2,3,4} output-of-partarred + rm -rf uncompressed-tars gzipped-tars + rm -rf *.partar{1,2,3,4}_diff + rm -rf *.partar{1,2,3,4}_done + rm -rf excld.dir.*tar excld.test + rm -rf excld-equal.dir.*tar excld-equal.test + if test -n "$$(find . -maxdepth 1 -name 'parrm_eaccess_test*_dir' -print -quit)"; then chmod -R u+rwx parrm_eaccess_test*_dir; fi + rm -rf parrm_eaccess_test*{.out,_dir} + rm -rf error.test.out error.tar linklink.tar linklink.partar linklink.tdir linklink.pdir linklink.txt excld.odir.partar excld.odir.tar + rm -rf permissions.in.dir permissions.tar.out.dir permissions.partar.out.dir permissions.tar.ls permissions.partar.ls + rm -rf *.ls *.test *.tar_test + rm -rf *.partar_out *.tar_out test1.out *.out + rm -rf test_fin + rm -f *.gcda *.gcno common/*.gcda common/*.gcno + rm -rf *.dir/ + rm -rf *.1.gz + rm -f $(COMMON_O) + rm -rf partar_log + +.PHONY: realclean +realclean: clean diff --git a/software/fss-parallel-tools/atdiff.c b/software/fss-parallel-tools/atdiff.c new file mode 100644 index 0000000..4c25bf0 --- /dev/null +++ b/software/fss-parallel-tools/atdiff.c @@ -0,0 +1,214 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include "putils.h" +#include "malloc.h" +/* parallel diff of a directory heirarchy that also checks ownership and attributes */ + +struct diff_task { + char *relative_path; + struct diff_task *next; +}; + +// Worker state +pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; ; +pthread_cond_t cond = PTHREAD_COND_INITIALIZER; +struct diff_task *tasks = NULL; +enum { n_workers = 32}; +size_t n_working = 0; // When tasks==NULL and n_working==0, we are done +pthread_t workers[n_workers]; + +#define PMAX 4096 +const char *src_dir; +const char *dst_dir; + +int check_ownership = 1; + +// From the qsort man page: +static int cmpstringp(const void *p1, const void *p2) { + // The actual arguments to this function are "pointers to pointers + // to char", but strcmp(3) arguments are "pointers to char", hence + // the following cast plus dereference. + return strcmp(* (char * const *) p1, * (char * const *) p2); +} + +static void collect_and_sort_pathnames(const char *abs_path, + // output: + size_t *np, + char ***paths_p) +// Collect all the pathnames in a directory, except for . and .., and +// return them in sorted order. + { + size_t n = 0; + size_t limit = 2; + char **MALLOC_N(paths, limit); + DIR *dir = opendir(abs_path); + assert(dir); + struct dirent *dentry; + while ((dentry = readdir(dir))) { + if (strcmp(dentry->d_name, ".") == 0) continue; + if (strcmp(dentry->d_name, "..") == 0) continue; + if (n >= limit) { + limit *= 2; + REALLOC(paths, limit); + } + assert(n < limit); + paths[n++] = strdup(dentry->d_name); + } + closedir(dir); + qsort(paths, n, sizeof(char*), cmpstringp); + *np = n; + *paths_p = paths; +} + +static void diff_r(const char *relative_path) { + char src_path[PMAX]; + { + int n = snprintf(src_path, PMAX, "%s/%s", src_dir, relative_path); + assert(n < PMAX); + } + char dst_path[PMAX]; + { + int n = snprintf(dst_path, PMAX, "%s/%s", dst_dir, relative_path); + assert(n < PMAX); + } + struct stat src_stat, dst_stat; + { + int src_r = lstat(src_path, &src_stat); + assert (src_r == 0); + } + { + int dst_r = lstat(dst_path, &dst_stat); + assert(dst_r == 0); + } + assert((src_stat.st_mode & S_IFMT) == (dst_stat.st_mode & S_IFMT)); + { + mode_t mask = 0777; + if ((src_stat.st_mode & mask) != (dst_stat.st_mode & mask)) { + fprintf(stderr, "masks differ %s and %s\n", src_path, dst_path); + assert(0); + } + } + if (check_ownership) { + if (src_stat.st_uid != dst_stat.st_uid) { + fprintf(stderr, "uid doesn't match for %s and %s\n", src_path, dst_path); + } + assert(src_stat.st_uid == dst_stat.st_uid); + assert(src_stat.st_gid == dst_stat.st_gid); + } + if (S_ISREG(dst_stat.st_mode)) { + assert(src_stat.st_size == dst_stat.st_size); + FILE *src_f = fopen(src_path, "r"); + FILE *dst_f = fopen(dst_path, "r"); + if (src_f == NULL || dst_f == NULL) { + fprintf(stderr, "Failed to open one or more files\n"); + return; + } + for (size_t i = 0 ; i < (size_t)(src_stat.st_size); i++) { + int src_c = getc(src_f); assert(src_c != EOF); + int dst_c = getc(dst_f); assert(dst_c != EOF); + assert(src_c == dst_c); + } + fclose(src_f); + fclose(dst_f); + } else if (S_ISDIR(src_stat.st_mode)) { + size_t sn, dn; + char **sps, **dps; + collect_and_sort_pathnames(src_path, &sn, &sps); + collect_and_sort_pathnames(dst_path, &dn, &dps); + assert(sn == dn); + for (size_t i = 0; i < sn; i++) { + assert(strcmp(sps[i], dps[i]) == 0); + struct diff_task *MALLOC(task); + char sub_rel_path[PMAX]; + { + int n = snprintf(sub_rel_path, PMAX, "%s/%s", relative_path, sps[i]); + assert(n < PMAX); + } + char *srp = strdup(sub_rel_path); + pthread_mutex_lock(&mutex); + *task = (struct diff_task){.relative_path = srp, + .next = tasks}; + tasks = task; + pthread_cond_signal(&cond); + pthread_mutex_unlock(&mutex); + } + } else if (S_ISLNK(src_stat.st_mode)) { + char slname[PMAX]; + ssize_t slen = readlink(src_path, slname, PMAX); + assert(slen >= 0 && slen < PMAX); + char dlname[PMAX]; + ssize_t dlen = readlink(dst_path, dlname, PMAX); + assert(dlen >= 0 && dlen < PMAX); + assert(slen == dlen); + slname[slen] = 0; + dlname[dlen] = 0; + if (0 != strcmp(slname, dlname)) { + fprintf(stderr, "%s -> %s is not the same as %s -> %s\n", src_path, slname, dst_path, dlname); + assert(0); + } + } +} + +static void* run_worker(void *d) { + pthread_mutex_lock(&mutex); + while (1) { + // Lock is held here. + while (!tasks && n_working) { + pthread_cond_wait(&cond, &mutex); + } + // Either there is a task or n_working==0 + if (tasks) { + struct diff_task *task = tasks; + tasks = task->next; + n_working++; + pthread_mutex_unlock(&mutex); + diff_r(task->relative_path); + free(task->relative_path); + free(task); + pthread_mutex_lock(&mutex); + n_working--; + if (tasks == NULL && n_working == 0) { + pthread_cond_broadcast(&cond); + } + } else if (n_working == 0) { + // No tasks and nothing to do. + pthread_mutex_unlock(&mutex); + return d; + } + } +} + +static void run_workers(void) { + MALLOC(tasks); + *tasks = (struct diff_task){.relative_path = strdup(""), + .next = NULL}; + n_working = 0; + for (size_t i = 0; i < n_workers; i++) { + pthread_create(&workers[i], NULL, run_worker, &workers[i]); + } + for (size_t i = 0; i < n_workers; i++) { + void *result; + pthread_join(workers[i], &result); + } +} + +int main (int argc, const char *argv[]) { + assert(argc == 3); + src_dir = argv[1]; + dst_dir = argv[2]; + run_workers(); + return 0; +} diff --git a/software/fss-parallel-tools/common/assert.c b/software/fss-parallel-tools/common/assert.c new file mode 100644 index 0000000..0119e66 --- /dev/null +++ b/software/fss-parallel-tools/common/assert.c @@ -0,0 +1,101 @@ +#include "assert.h" +#include // for PRIx64 +#include // for size_t +#include // for uint64_t +#include // for abort +#include // for va_end +#include "backtrace.h" // for spew_backtrace +#include "spew.h" // for log0, ::SPEW_FATAL, the_spew_log +#include "../version.h" + +void assert_fail(const char *assertion, + const char *file, size_t line, const char *function) +{ + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, + "ASSERTION FAILURE: %s", assertion); + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, "GITREV %s", gitrev); + spew_backtrace(SPEW_FATAL); + abort(); +} + + +void assert_fail_int(const char *assertion, const char *str_var, int var, + const char *file, size_t line, const char *function) +{ + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, "ASSERTION FAILURE: %s, %s=%d", + assertion, str_var, var); + spew_backtrace(SPEW_FATAL); + abort(); +} + + +void assert_fail_v(const char *assertion, + const char *file, + size_t line, + const char *function, + const char *fmt_string, + ...) { + va_list args; +#ifndef __CHECKER__ + // va_start() contains a cast to void * (from char const **, in + // this case), which runs afoul of sparse's check for qualifier + // drops. + // + va_start(args, fmt_string); +#endif // !__CHECKER__ + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, + "ASSERTION FAILURE: %s", assertion); + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, "GITREV %s", gitrev); + logv(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, fmt_string, args); + spew_backtrace(SPEW_FATAL); + abort(); + va_end(args); +} + +void check_eq_u(uintmax_t a, uintmax_t b, + const char *file, size_t line, const char *function, + const char *assertion) +{ + if (a != b) { + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, + "ASSERTION FAILURE: [%s]: 0x%"PRIxMAX" != 0x%"PRIxMAX, + assertion, a, b); + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, "GITREV %s", gitrev); + spew_backtrace(SPEW_FATAL); + abort(); + } +} + +void check_eq_i(intmax_t a, intmax_t b, + const char *file, size_t line, const char *function, + const char *assertion) +{ + if (a != b) { + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, + "ASSERTION FAILURE [%s]: %"PRIiMAX" != %"PRIiMAX, + assertion, a, b); + log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1, + /*is_stack_trace=*/0, + SPEW_FATAL, file, line, function, "GITREV %s", gitrev); + spew_backtrace(SPEW_FATAL); + abort(); + } +} diff --git a/software/fss-parallel-tools/common/assert.h b/software/fss-parallel-tools/common/assert.h new file mode 100644 index 0000000..a18ded3 --- /dev/null +++ b/software/fss-parallel-tools/common/assert.h @@ -0,0 +1,98 @@ +#ifndef __COMMON_ASSERT_H__ +#define __COMMON_ASSERT_H__ + +#include // for size_t +#include // for uint64_t +#include "simplest-defs.h" // for STRINGIFY + +// Use CHECK for checking something that you really want checked. +// +// Use ASSERT for checking something that you want checked, but the +// functionality of the program won't be compromised if the predicate isn't +// checked, or even run. It should be correct to define ASSERT as "#define +// ASSERT(x) ((void)0)". This means that you should not put side effects into +// ASSERT predicates. +// +// Use ASSERT_WITH_SIDE_EFFECTS for checking something you want checked, but +// there may be side effects in the predicate, so we need need to evaluate the +// predicate even if NOASSERT is set. +// +// See wisdom.tex for more discussion. + +#ifndef CHECK_ASSERT_SIDE_EFFECTS +#define CHECK_ASSERT_SIDE_EFFECTS 0 +#endif + +#ifdef NOASSERT + #define ASSERT(x) ((0) ? ((void)(x), (void)(0)) : (void)(0)) + #define ASSERT0 ASSERT + #define ASSERTv(x, typeof_val, val) ((0) ? ((void)(x), (void)(val), (void)(0)) : (void)(0)) + #define ASSERT_EQ_u(a, b) ((0) ? ((void)(a), (void)(b), (void)(0)) \ + : (void)(0)) + #define ASSERT_EQ_i(a, b) ((0) ? ((void)(a), (void)(b), (void)(0)) \ + : (void)(0)) + #define ASSERT_WITH_SIDE_EFFECTS(expr) ((void)expr) +#elif !CHECK_ASSERT_SIDE_EFFECTS // && !NOASSERT + #define ASSERT CHECK + #define ASSERT0 CHECK0 + #define ASSERTv CHECKv + #define ASSERT_EQ_u CHECK_EQ_u + #define ASSERT_EQ_i CHECK_EQ_i + #define ASSERT_WITH_SIDE_EFFECTS ASSERT +#else // CHECK_ASSERT_SIDE_EFFECTS && !NOASSERT + // See https://stackoverflow.com/questions/10593492/catching-assert-with-side-effects + // for this trick for noticing that you have an ASSERT with side effects. + extern int not_needed_if_expr_has_no_side_effects; + #define ASSERT(expr) ((void)(not_needed_if_expr_has_no_side_effects || (expr))) + #define ASSERT0(expr) ASSERT((expr) == 0) + #define ASSERTv(x, typeof_val, val) ASSERT(x) + #define ASSERT_EQ_u(a, b) ASSERT((a) == (b)) + #define ASSERT_EQ_i(a, b) ASSERT((a) == (b)) + #define ASSERT_WITH_SIDE_EFFECTS(expr) ((void)(expr)) +#endif // CHECK_ASSERT_SIDE_EFFECTS && !NOASSERT + +#define CHECK(x) ((x) ? (void) (0) : assert_fail (STRINGIFY(x), __FILE__, __LINE__, __PRETTY_FUNCTION__)) +#define CHECK0(x) CHECK((x) == 0) +/* Check and print variable of specified type. */ +#define CHECKv(x, typeof_val, val) ((x) ? (void) (0) : assert_fail_ ## typeof_val \ + (STRINGIFY(x), #val, val, __FILE__, __LINE__, __PRETTY_FUNCTION__)) +#define CHECKvv(x, fmt_args...) ((x) ? (void)(0) : assert_fail_v(STRINGIFY(x), __FILE__, __LINE__, __PRETTY_FUNCTION__, fmt_args) ) +#define UNREACHABLE() CHECK("UNREACHABLE" && 0) + +void assert_fail(const char *assertion, + const char *file, size_t line, const char *function) + __attribute__ ((__noreturn__)); +void assert_fail_int(const char *assertion, const char *str_var, int var, + const char *file, size_t line, const char *function) + __attribute__ ((__noreturn__)); +void assert_fail_v(const char *assertion, + const char *file, + size_t line, + const char *function, + const char *fmt_string, + ...) + __attribute__ ((__noreturn__)) + __attribute__ ((format (printf, 5, 6))); +void +check_eq_u(uintmax_t a, uintmax_t b, + const char *file, size_t line, const char *function, + const char *assertion); + +#define CHECK_EQ_u(a, b) ( \ + check_eq_u((uintmax_t)(a), (uintmax_t)(b), \ + __FILE__, __LINE__, __PRETTY_FUNCTION__, \ + STRINGIFY(a) " == " STRINGIFY(b))) + + +void +check_eq_i(intmax_t a, intmax_t b, + const char *file, size_t line, const char *function, + const char *assertion); + +#define CHECK_EQ_i(a, b) ( \ + check_eq_i((intmax_t)(a), (intmax_t)(b), \ + __FILE__, __LINE__, __PRETTY_FUNCTION__, \ + STRINGIFY(a) " == " STRINGIFY(b))) + + +#endif diff --git a/software/fss-parallel-tools/common/backtrace.c b/software/fss-parallel-tools/common/backtrace.c new file mode 100644 index 0000000..f49124e --- /dev/null +++ b/software/fss-parallel-tools/common/backtrace.c @@ -0,0 +1,347 @@ +#define _GNU_SOURCE +#include "backtrace.h" +#include // for backtrace, backtrace_symbols +#include // for open, O_CLOEXEC, O_RDONLY +#include // for pthread_mutex_lock, pthread_mute... +#include // for posix_spawn_file_actions_t, posi... +#include // for ptrdiff_t +#include // for snprintf, sscanf +#include // for size_t, NULL, abort, free, strtoull +#include // for memset, strrchr +#include // for waitpid +#include // for close, read, pipe2, ssize_t, rea... +#include "simplest-defs.h" // for NELEM, UNUSED +#include "sparse.h" // for __force +#include "spew.h" // for spew, SPEW_DEBUG, SPEW_INFO, SPE... + +/* DO NOT USE ASSERT() IN THIS FILE, TO AVOID INFINITE RECURSION */ +#ifdef ASSERT +#error CANNOT_USE_ASSERT_IN_THIS_FILE +#endif + +// I'm using pthread_mutex_t rather than xthread_mutex_t because I cannot +// think of a better way to do this than with a static variable. And with a +// static variable, I want to use PTHREAD_MUTEX_INITIALIZER, which there +// doesn't seem to have a xthread_mutex analogue. + +static pthread_mutex_t backtrace_mutex = PTHREAD_MUTEX_INITIALIZER; + +// Backtracer wants to fork an "addr2line" process to be able to translate +// code offsets into human-readable strings. Back in the dark ages, we would +// do the fork whenever we wanted a backtrace. That turned out to be too +// expensive, since fork() would copy huge page table (covering many gigabytes +// of memory), and it was silly because the exec() would just throw it away. +// +// Our next solution used a constructor/destructor. At the beginning of the +// program we would start up the backtracer. At that point in time the page +// table was small, and we didn't really care if it was expensive since +// nothing was really happening yet. However we don't really like the +// constructor/destructor stuff, since it's fragile. (And Matteo doesn't like +// it because it's a desugared version of C++ :-) +// +// This solution uses posix_spawnp() to do a fork/exec operation. +// posix_spawnp() lets you do some housekeeping (with dup2 and close +// operations) between the fork and the spawn. Because the kernel gets all +// the information about the fork/dup/close/spawn sequence at once, it can +// avoid copying the page tables. The performance of this sequence is about +// 200us to 500us. + +struct backtracer { + int broken; // true if something went wrong and the backtracer + // is failing. Fall back to not printing + // backtraces, and try to keep running. + pid_t child; // The addr2line process + int pipe_to_child; // The write end of a pipe that goes the + // addr2line's stdin. + int pipe_from_child; // The read end of a pipe that comes from + // addr2line's stdout. + size_t va_base; // The offset to be subtracted from addresses to + // to ASLR. +}; + +static int __attribute__((warn_unused_result)) // NEGATIVE_ON_ERROR +sinit(posix_spawn_file_actions_t *file_actions) +// Effect: Initialize the file_actions. If anything goes wrong return -1. +{ + int r = posix_spawn_file_actions_init(file_actions); + if (r != 0) { + spew(SPEW_DEBUG, "Cannot spawn_file_actions_init: r=%d", r); + return -1; + } else { + return 0; + } + } + +static int __attribute__((warn_unused_result)) // NEGATIVE_ON_ERROR +sdup2(posix_spawn_file_actions_t *file_actions, int from, int to) +// Effect: Record a dup2() operation to be done when posix_spawn runs. If +// anything goes wrong, spew a diagnostic and return -1; +{ + int r = posix_spawn_file_actions_adddup2(file_actions, from, to); + if (r != 0) { + spew(SPEW_DEBUG, "Cannot spawn_fileactions_adddup2: r=%d", r); + return -1; + } else { + return 0; + } +} + +static int __attribute__((warn_unused_result)) // NEGATIVE_ON_ERROR +sclose(posix_spawn_file_actions_t *file_actions, int fd) +// Effect: Record a close() operation to be cone when posix_spawn runs. If +// anything goes wrong, spew a diagnostic and return -1; +{ + int r = posix_spawn_file_actions_addclose(file_actions, fd); + if (r != 0) { + spew(SPEW_DEBUG, "Cannot spawn_fileactions_addclose: r=%d", r); + return -1; + } else { + return 0; + } +} + +static const struct backtracer errresult = { + .pipe_to_child=-1, .pipe_from_child=-1, .child=0, .broken=1, .va_base = 0}; + +static struct backtracer +mk_backtracer(void) { + // Only one backtracer may exist at a time (to prevent forkbombing if all + // threads start backtracing.) This mutex is unlocked when you destroy + // the backtracer. + pthread_mutex_lock(&backtrace_mutex); + + // Plumbing + int tochildpipe[2]; + int fromchildpipe[2]; + { + int r = pipe2(tochildpipe, O_CLOEXEC); + if (r) { + spew(SPEW_ERROR, "pipe2 failed"); + } + } + { + int r = pipe2(fromchildpipe, O_CLOEXEC); + if (r) { + spew(SPEW_ERROR, "pipe2 failed"); + } + } + + // Find the path to the current executable. It's a symlink in + // /proc/self/exe. + + char exec_path[204]; + // readlink doesn't null terminate. Fill it with zeros and tell it to + // only use sizeof(exec_path)-1 to make sure there is a null at the + // end. + memset(exec_path, 0, sizeof(exec_path)); + ssize_t rc = readlink("/proc/self/exe", exec_path, + sizeof(exec_path)-1); + UNUSED(rc); + spew(SPEW_INFO, "exec path is %s", exec_path); + + // Set up to do the plumbing after the spawn. + posix_spawn_file_actions_t file_actions; + if (0 + || sinit(&file_actions) + || sdup2(&file_actions, tochildpipe[0], 0) + || sdup2(&file_actions, fromchildpipe[1], 1) + || sclose(&file_actions, tochildpipe[0]) + || sclose(&file_actions, tochildpipe[1]) + || sclose(&file_actions, fromchildpipe[0]) + || sclose(&file_actions, fromchildpipe[1])) + { + return errresult; + } + pid_t child; + { + // Hack around because posix_spawnp doesn't have const's in quite the right place. + char addr2line_s[] = "addr2line"; + char _e_s[] = "-e"; + char _f_s[] = "-f"; + char * const argv[] = {addr2line_s, _e_s, exec_path, _f_s, NULL}; + int r = posix_spawnp(&child, "addr2line", + &file_actions, NULL, + argv, environ); + if (r != 0) { + spew(SPEW_DEBUG, "Could not posix_spawnp r=%d", r); + return errresult; + } + } + { + int r = posix_spawn_file_actions_destroy(&file_actions); + if (r) spew(SPEW_FATAL, "Cannot destroy fail_actions r=%d", r); + } + + // Plumbing back here the main program. + struct backtracer result = {.broken = 0, + .child = child, + .pipe_to_child = tochildpipe[1], + .pipe_from_child = fromchildpipe[0], + .va_base = 0}; + close(tochildpipe[0]); + close(fromchildpipe[1]); + + // Find the va_base + { + int fd = open("/proc/self/maps", O_RDONLY); + if (fd >= 0) { + char addr_buffer[256]; + // HACK: we're going to assume that the first line in the + // file contains the mapping for our executable. Dynamic + // libraries will be mapped in different locations, but we + // don't really use any of them so our traceback should be + // mostly correct. + ssize_t bytes_read = read(fd, &addr_buffer, NELEM(addr_buffer)-1); + if (bytes_read > 0) { + addr_buffer[bytes_read] = 0; + int rval = sscanf(addr_buffer, "%zx-", &result.va_base); + if (rval != 1) { + result.va_base = 0; + } + } + } + close(fd); + } + spew(SPEW_INFO, "va_base is %zx", result.va_base); + if (result.va_base < 0x800000) { + // If ASLR is off, subtracting va_base makes offsets that addr2line + // doesn't understand. The best heuristic I've (jmp) found is that if + // va_base is "low", then ASLR is off. I haven't been able to figure + // out if there's a "low" address under which ASLR is disabled, but + // I've seen 4 MiB in many systems, so I'm going to claim any va_base + // under 8 MiB is in a non-ASLR system. + result.va_base = 0; + } + + return result; +} + +static void +backtracer_destroy(struct backtracer *bt) { + close(bt->pipe_to_child); + close(bt->pipe_from_child); + spew(SPEW_INFO, "waitpid(%d)", bt->child); + int wstatus; + waitpid(bt->child, &wstatus, 0); + spew(SPEW_INFO, "waitpid(%d) done wstatus=%d", bt->child, wstatus); + *bt = errresult; + pthread_mutex_unlock(&backtrace_mutex); +} + +static void read_tail_of_line(int fd, char *line, size_t linelen) +// Effect: Read a line of data from fd into line. If the line is too long to +// fit, then put the last characters in so that if we read +// "longfilename/fun.c:25" and linelen is 11 we end up with "e/fun.c:25" +// rather than "longfilena". +// line will always end NUL-terminated. +// Requires: linelen>0. This function will fail unpredictably if linelen==0. +{ + for (size_t i = 0; i+1 < linelen; i++) { + char c; + ssize_t rr = read(fd, &c, 1); + if (0 == rr || '\n' == c) { + line[i] = 0; + return; + } else { + line[i] = c; + } + } + while (1) { + char c; + ssize_t rr = read(fd, &c, 1); + if (0 == rr || '\n' == c) { + line[linelen-1] = 0; + return; + } else { + line[linelen-1] = c; + // Shift the string left by 1.o + for (size_t j = 0; j+1 < linelen; j++) { + line[j] = line[j+1]; + } + } + } +} + +static void spew_backtrace_symbol(struct backtracer *bt, + const void *addr, + const char *from_backtrace_symbol, + size_t spew_level) +{ + void *addr_from_base = (void*)((__force size_t)addr + - bt->va_base); + char str_of_ptr[20]; + char fun_line[300] = ""; + char file_line[300] = ""; + if (!bt->broken) { + int len = snprintf(str_of_ptr, sizeof(str_of_ptr), + "%p\n", addr_from_base); + ssize_t wr = write(bt->pipe_to_child, str_of_ptr, + (__force size_t)(ptrdiff_t)len); + int did_write = wr == len; + if (!did_write) { + bt->broken = 1; + } else { + read_tail_of_line(bt->pipe_from_child, fun_line, sizeof(fun_line)); + read_tail_of_line(bt->pipe_from_child, file_line, sizeof(file_line)); + } + } + if (!from_backtrace_symbol) { + from_backtrace_symbol = "(unknown offset)"; + } + const char* function = &fun_line[0]; + if (!fun_line[0]) { + function = "(unknown function)"; + } + size_t line_number = 0; + const char* filename = &file_line[0]; + if (*filename) { + // Find line number by looking for last : in file_line. + char* line_number_start = strrchr(file_line, ':'); + if (line_number_start != NULL) { + // Parse the line number + line_number = strtoull(line_number_start + 1, NULL, 10); + if (line_number > 0) { + // Parsed, remove the line number and : from file_line. + *line_number_start = 0; + } + } + } else { + // No filename, substitute backtrace symbol for filename. + filename = from_backtrace_symbol; + from_backtrace_symbol = ""; + } + spew_for_backtrace( + spew_level, filename, line_number, function, from_backtrace_symbol); +} + +void spew_stored_backtrace(void *const*ptrs, int nptrs, size_t spew_level) +{ + char **strings = backtrace_symbols(ptrs, nptrs); + + struct backtracer bt = mk_backtracer(); + + for (int j = 0; j < nptrs; j++) { + spew_backtrace_symbol(&bt, + ptrs[(__force ptrdiff_t)j], + strings ? strings[(__force ptrdiff_t)j] : NULL, + spew_level); + } + + backtracer_destroy(&bt); + + free(strings); +} + +void spew_backtrace(size_t spew_level) +{ + if (spew_level < spew_get_level()) return; + void *buffer[100]; + int nptrs = backtrace(buffer, (__force int)NELEM(buffer)); + + /* cannot assert in this file */ + if (nptrs >= 0 && nptrs <= (__force int)NELEM(buffer)) { + spew_stored_backtrace(buffer, nptrs, spew_level); + } else { + abort(); // can't happen + } +} diff --git a/software/fss-parallel-tools/common/backtrace.h b/software/fss-parallel-tools/common/backtrace.h new file mode 100644 index 0000000..fc9ea2e --- /dev/null +++ b/software/fss-parallel-tools/common/backtrace.h @@ -0,0 +1,16 @@ +#ifndef __BACKTRACE_H__ +#define __BACKTRACE_H__ + +#include // For size_t + +/* Fancier backtrace facility. + */ +void spew_backtrace(size_t spew_level); +void spew_stored_backtrace(void *const*ptrs, + // This is an int, rather than a size_t because + // I cannot ASSERT that nelem fits in an int + // inside the spew code. + int nelem, + size_t spew_level); + +#endif // !__SPEW_H__ diff --git a/software/fss-parallel-tools/common/debug-tag.c b/software/fss-parallel-tools/common/debug-tag.c new file mode 100644 index 0000000..2551b18 --- /dev/null +++ b/software/fss-parallel-tools/common/debug-tag.c @@ -0,0 +1,115 @@ +#include "debug-tag.h" +#include // for PRIx64 +#include // for snprintf, NULL, size_t + +static __thread struct debug_tag debug_tag = { + .origin = (uint64_t)DEBUG_TAG_UNKNOWN, + .tag = 0, + .reserved = 0, + .proto = 0, + .proc = 0, +}; + +static const char *debug_tag_name[] = { + [DEBUG_TAG_UNKNOWN] = "UNKNOWN", + [DEBUG_TAG_TEST] = "T", + [DEBUG_TAG_ORCA] = "OR", + [DEBUG_TAG_ORCA_AGENT] = "OA", + [DEBUG_TAG_MDDBG] = "MDDBG", +}; + +static const char *debug_tag_protocols[] = { + [DT_PROTO_UNKNOWN] = "UNKNOWN", + [DT_PROTO_RPCBIND] = "RPCBIND", + [DT_PROTO_MOUNT] = "MOUNT", + [DT_PROTO_NFSv3] = "NFS", + [DT_PROTO_NLM] = "NLM", + [DT_PROTO_MDD] = "MDD", +}; + +struct debug_tag debt_set(struct debug_tag dbt) +{ + struct debug_tag old = debug_tag; + debug_tag = dbt; + return old; +} + +struct debug_tag debt_get(void) +{ + return debug_tag; +} + +void debt_advance(void) +{ + debug_tag.proto = 0; + debug_tag.proc = 0; + debug_tag.tag++; +} + +void debt_ctx(enum debug_tag_protocol_e proto, uint8_t proc) +{ + debug_tag.proto = (uint8_t)proto; + debug_tag.proc = proc; +} + +static const char *origin_name(uint64_t origin) +{ + switch ((enum debug_tag_origin_e)origin) { + case DEBUG_TAG_UNKNOWN: + case DEBUG_TAG_TEST: + case DEBUG_TAG_ORCA: + case DEBUG_TAG_ORCA_AGENT: + case DEBUG_TAG_MDDBG: + return debug_tag_name[origin]; + default: + return "INVALID"; + } +} + +static const char *protocol_name(uint8_t proto) +{ + switch ((enum debug_tag_protocol_e)proto) { + case DT_PROTO_UNKNOWN: + case DT_PROTO_RPCBIND: + case DT_PROTO_MOUNT: + case DT_PROTO_NFSv3: + case DT_PROTO_NLM: + case DT_PROTO_MDD: + return debug_tag_protocols[(ptrdiff_t)proto]; + default: + return "INVALID"; + } +} + +void debt_snprintf_specific(struct debug_tag debt, char *buf, size_t size) +{ + if (debt.origin == (uint64_t)DEBUG_TAG_UNKNOWN) { + buf[0] = 0; + } else { + if (debt.proto == 0 && debt.proc == 0) { + snprintf(buf, size, "#%s/%"PRIx32":", + origin_name(debt.origin), + debt.tag); + } else { + snprintf(buf, size, "#%s/%"PRIx32"/%s/%"PRIx8":", + origin_name(debt.origin), + debt.tag, + protocol_name(debt.proto), + debt.proc); + } + } +} + +void debt_snprintf(char *buf, size_t size) +{ + debt_snprintf_specific(debug_tag, buf, size); +} + +void debt_snprintf_concise(char *buf, size_t size) +{ + if (debug_tag.origin == (uint64_t)DEBUG_TAG_UNKNOWN) { + buf[0] = 0; + } else { + snprintf(buf, size, "%"PRIx32, debug_tag.tag); + } +} diff --git a/software/fss-parallel-tools/common/debug-tag.h b/software/fss-parallel-tools/common/debug-tag.h new file mode 100644 index 0000000..f39c239 --- /dev/null +++ b/software/fss-parallel-tools/common/debug-tag.h @@ -0,0 +1,44 @@ +#ifndef ___DEBUG_TAG_H___ +#define ___DEBUG_TAG_H___ + +#include // for size_t +#include // for uint32_t, uint64_t +#include "sparse.h" // for __ffsw_bitwise +#include "simplest-defs.h" // for ENUM_AS_UINT64, __enum64 + +__ffsw_bitwise enum debug_tag_origin_e { + DEBUG_TAG_UNKNOWN = (uint64_t)0, + DEBUG_TAG_TEST = (uint64_t)1, + DEBUG_TAG_ORCA = (uint64_t)2, + DEBUG_TAG_ORCA_AGENT = (uint64_t)3, + DEBUG_TAG_MDDBG = (uint64_t)4, +} __enum64; + +__ffsw_bitwise enum debug_tag_protocol_e { + DT_PROTO_UNKNOWN = (uint64_t)0, + DT_PROTO_RPCBIND = (uint64_t)1, + DT_PROTO_MOUNT = (uint64_t)2, + DT_PROTO_NFSv3 = (uint64_t)3, + DT_PROTO_NLM = (uint64_t)4, + DT_PROTO_MDD = (uint64_t)5, +} __enum64; + +struct debug_tag { + uint64_t origin; + uint32_t tag; + uint16_t reserved; + uint8_t proto; + uint8_t proc; +}; + +STATIC_ASSERT(sizeof(struct debug_tag) == 16); + +struct debug_tag debt_set(struct debug_tag dbt); +struct debug_tag debt_get(void); +void debt_advance(void); +void debt_ctx(enum debug_tag_protocol_e proto, uint8_t proc); +void debt_snprintf(char *buf, size_t size); +void debt_snprintf_specific(struct debug_tag debt, char *buf, size_t size); +void debt_snprintf_concise(char *buf, size_t); + +#endif diff --git a/software/fss-parallel-tools/common/error.c b/software/fss-parallel-tools/common/error.c new file mode 100644 index 0000000..7b79609 --- /dev/null +++ b/software/fss-parallel-tools/common/error.c @@ -0,0 +1,168 @@ +#include "error.h" +#include // for E2BIG, EACCES, EADDRINUSE, EADDRNOTAVAIL +#include // for ptrdiff_t, size_t +#include "sparse.h" // for __force +#include "simplest-defs.h" // for NELEM + +/* One is not supposed to use sys_errlist[] any longer. + + strerror() grabs locks and loads a locale to translate the error + message. On top of that, it's not thread safe. + + To work around the thread-safe business, one is supposed to + allocate a buffer and call strerror_r, for the privilege of having + locale-dependent error messages precisely when things are going + wrong and the last thing I want to do is load a file from disk. + + Oracle policy prevents me from drawing the appropriate conclusion + in writing. Let's just say that returning a static string seems + simpler than any of the above. +*/ +/* as long as I am redoing strerror(), we may as well print + ECONNABORTED instead of some translation into English, which then I + have to mentally map back to ECONNABORTED */ +#define DO(x) [x] = #x +static const char *const errlist[] = { + DO(EPERM), + DO(ENOENT), + DO(ESRCH), + DO(EINTR), + DO(EIO), + DO(ENXIO), + DO(E2BIG), + DO(ENOEXEC), + DO(EBADF), + DO(ECHILD), + DO(EDEADLK), + DO(ENOMEM), + DO(EACCES), + DO(EFAULT), + DO(ENOTBLK), + DO(EBUSY), + DO(EEXIST), + DO(EXDEV), + DO(ENODEV), + DO(ENOTDIR), + DO(EISDIR), + DO(EINVAL), + DO(EMFILE), + DO(ENFILE), + DO(ENOTTY), + DO(ETXTBSY), + DO(EFBIG), + DO(ENOSPC), + DO(ESPIPE), + DO(EROFS), + DO(EMLINK), + DO(EPIPE), + DO(EDOM), + DO(ERANGE), + DO(EAGAIN), + DO(EINPROGRESS), + DO(EALREADY), + DO(ENOTSOCK), + DO(EMSGSIZE), + DO(EPROTOTYPE), + DO(ENOPROTOOPT), + DO(EPROTONOSUPPORT), + DO(ESOCKTNOSUPPORT), + DO(EOPNOTSUPP), + DO(EPFNOSUPPORT), + DO(EAFNOSUPPORT), + DO(EADDRINUSE), + DO(EADDRNOTAVAIL), + DO(ENETDOWN), + DO(ENETUNREACH), + DO(ENETRESET), + DO(ECONNABORTED), + DO(ECONNRESET), + DO(ENOBUFS), + DO(EISCONN), + DO(ENOTCONN), + DO(EDESTADDRREQ), + DO(ESHUTDOWN), + DO(ETOOMANYREFS), + DO(ETIMEDOUT), + DO(ECONNREFUSED), + DO(ELOOP), + DO(ENAMETOOLONG), + DO(EHOSTDOWN), + DO(EHOSTUNREACH), + DO(ENOTEMPTY), + DO(EUSERS), + DO(EDQUOT), + DO(ESTALE), + DO(EREMOTE), + DO(ENOLCK), + DO(EILSEQ), + DO(EBADMSG), + DO(EIDRM), + DO(EMULTIHOP), + DO(ENODATA), + DO(ENOLINK), + DO(ENOMSG), + DO(ENOSR), + DO(ENOSTR), + DO(EOVERFLOW), + DO(EPROTO), + DO(ETIME), + DO(ECANCELED), + DO(ECHRNG), + DO(EL2NSYNC), + DO(EL3HLT), + DO(EL3RST), + DO(ELNRNG), + DO(EUNATCH), + DO(ENOCSI), + DO(EL2HLT), + DO(EBADE), + DO(EBADR), + DO(EXFULL), + DO(ENOANO), + DO(EBADRQC), + DO(EBADSLT), + DO(EBFONT), + DO(ENONET), + DO(ENOPKG), + DO(EADV), + DO(ESRMNT), + DO(ECOMM), + DO(EDOTDOT), + DO(ENOTUNIQ), + DO(EBADFD), + DO(EREMCHG), + DO(ELIBACC), + DO(ELIBBAD), + DO(ELIBSCN), + DO(ELIBMAX), + DO(ELIBEXEC), + DO(ESTRPIPE), + DO(EUCLEAN), + DO(ENOTNAM), + DO(ENAVAIL), + DO(EISNAM), + DO(EREMOTEIO), + DO(ENOMEDIUM), + DO(EMEDIUMTYPE), + DO(ENOKEY), + DO(EKEYEXPIRED), + DO(EKEYREVOKED), + DO(EKEYREJECTED), + DO(EOWNERDEAD), + DO(ENOTRECOVERABLE), + DO(ERFKILL), + DO(EHWPOISON), +}; + +const char *xstrerror(int err) +{ + ptrdiff_t err1 = err; + if (1 + && err1 >= 0 + && (__force size_t)err1 < NELEM(errlist) + && errlist[err1]) { + return errlist[err1]; + } else { + return "Unknown error"; + } +} diff --git a/software/fss-parallel-tools/common/error.h b/software/fss-parallel-tools/common/error.h new file mode 100644 index 0000000..6d75a28 --- /dev/null +++ b/software/fss-parallel-tools/common/error.h @@ -0,0 +1,6 @@ +#ifndef __ERROR_H__ +#define __ERROR_H__ + +const char *xstrerror(int err); + +#endif diff --git a/software/fss-parallel-tools/common/simplest-defs.h b/software/fss-parallel-tools/common/simplest-defs.h new file mode 100644 index 0000000..2e70c06 --- /dev/null +++ b/software/fss-parallel-tools/common/simplest-defs.h @@ -0,0 +1,70 @@ +#ifndef __SIMPLEST_DEFS_H__ +#define __SIMPLEST_DEFS_H__ + + +#include +#include + +#include "sparse.h" + + +/* FIXME [matteof 2017-06-16] The name of this type is wrong. */ + +/* BOOLINT intends to provide a type that can be stored in + structs without worrying about alignment, and is compatible + with C's rules, so that + + boolint foo = (1 != 0); + foo && (...) + + work. +*/ +typedef ptrdiff_t boolint; +#define FMTboolint "%td" + +#if defined(__clang__) +#define __enum64 +#define ENUM_AS_UINT64(e) uint64_t +#else +#define __enum64 __attribute__((mode(DI))) +#define ENUM_AS_UINT64(e) e +#endif + +#define UNUSED(x) ((void)(x)) +#define STATIC_ASSERT(truth) \ + extern char \ + (*static_assert_test_array__(void))[sizeof(char[(2 * !!(truth)) - 1])] + +// See +// https://gcc.gnu.org/onlinedocs/gcc-4.7.0/cpp/Stringification.html +// for an explanation of why the extra level of macro expansion might +// be needed. +// +#define STRINGIFY(x) #x +#define XSTRINGIFY(x) STRINGIFY(x) + +#define SAME_TYPE(a, b) (__builtin_types_compatible_p(typeof(a), typeof(b))) + +#define POINTER_IS_ARRAY(a) (!SAME_TYPE((a), &(a)[0])) + +// Statically calculate size of array +// Provide compile error if +// - a is not a pointer +// - a is not an array (e.g. regular pointer, "array" function parameter) +// Casting to ssize_t is necessary because y and z in (x?y:z) need to be the +// same type and we need the -1 to get the compile error +// +// ({STATIC_ASSERT(POINTER_IS_ARRAY(a)); sizeof(a)/sizeof((a)[0]);}) +// does not work because our STATIC_ASSERT is not smart enough to be nested +// (NELEM is often used inside STATIC_ASSERT) +// +// The type of X in char[X] should be ptrdiff_t, but we want to avoid forcing +// the gratuitous inclusion of . "long" is good enough for the +// purpose. +// +#define NELEM(a) \ + (sizeof(char[POINTER_IS_ARRAY(a) ? \ + (__force long)(sizeof(a) / sizeof((a)[0])) : \ + (long)-1])) + +#endif // !__SIMPLEST_DEFS_H__ diff --git a/software/fss-parallel-tools/common/sparse.h b/software/fss-parallel-tools/common/sparse.h new file mode 100644 index 0000000..81041ad --- /dev/null +++ b/software/fss-parallel-tools/common/sparse.h @@ -0,0 +1,30 @@ +/* support for the sparse static analyzer */ + +#ifndef __SPARSE_H__ +#define __SPARSE_H__ + +#ifdef __CHECKER__ +#define __force __attribute__((force)) +/* Avoid name clash with __bitwise defined in linux/types.h. */ +#define __ffsw_bitwise __attribute__((bitwise)) +#define __unitlike __attribute__((unitlike)) +#define __designated_init __attribute__((designated_init)) +#define __allow_by_value __attribute__((allow_by_value)) +#define __allow_large_assignment __attribute__((allow_large_assignment)) +#define __disallow_by_value __attribute__((disallow_by_value)) +#define NOT_CHECKERED(...) +#else /* __CHECKER__ */ +#define __force /**/ +#define __ffsw_bitwise /**/ +#define __unitlike /**/ +#define __designated_init /**/ +#define __allow_by_value /**/ +#define __allow_large_assignment /**/ +#define __disallow_by_value /**/ +#define __aligned_void void +#define __writeonly /**/ +#define __force_local_static /**/ +#define NOT_CHECKERED(...) __VA_ARGS__ +#endif /* __CHECKER__ */ + +#endif /* __SPARSE_H__ */ diff --git a/software/fss-parallel-tools/common/spew-throttled.h b/software/fss-parallel-tools/common/spew-throttled.h new file mode 100644 index 0000000..3cae39f --- /dev/null +++ b/software/fss-parallel-tools/common/spew-throttled.h @@ -0,0 +1,50 @@ +#ifndef FFSW_SPEW_THROTTLED_H +#define FFSW_SPEW_THROTTLED_H + +#include // for size_t +#include // for pthread_mutex_lock, pthread_mute... +#include "common/tick.h" // for tickns + +struct spew_throttle { + pthread_mutex_t mx; + const tickns_t period; + tickns_t last_suppressed; + size_t n_suppressed; +}; + +// To avoid the "static variable defined locally" warning +#ifndef __CHECKER__ +#define init_throttle(__spew_throttle,wait_time) \ + static struct spew_throttle __spew_throttle = { \ + .mx = PTHREAD_MUTEX_INITIALIZER, \ + .period = (wait_time), \ + .last_suppressed = 0, \ + .n_suppressed = 0, \ + }; +#else +#define init_throttle(obj,wait_time) \ + struct spew_throttle obj = { \ + .mx = PTHREAD_MUTEX_INITIALIZER, \ + .period = (wait_time), \ + .last_suppressed = 0, \ + .n_suppressed = 0, \ + }; +#endif // __CHECKER__ + +#define spew_throttled(wait_time, level, format, ...) \ + ({ \ + init_throttle(__spew_throttle, wait_time); \ + CHECK0(pthread_mutex_lock(&__spew_throttle.mx)); \ + tickns_t __now = tickns(); \ + if (__now >= (__spew_throttle.last_suppressed) + ( __spew_throttle.period)) { \ + __spew_throttle.last_suppressed = __now; \ + spew(level,format " (suppressed:%zu)", ##__VA_ARGS__, __spew_throttle.n_suppressed); \ + __spew_throttle.n_suppressed = 0; \ + } \ + else { \ + __spew_throttle.n_suppressed += 1; \ + } \ + CHECK0(pthread_mutex_unlock(&__spew_throttle.mx)); \ + }) + +#endif // FFSW_SPEW_THROTTLED_H diff --git a/software/fss-parallel-tools/common/spew.c b/software/fss-parallel-tools/common/spew.c new file mode 100644 index 0000000..7d84984 --- /dev/null +++ b/software/fss-parallel-tools/common/spew.c @@ -0,0 +1,1009 @@ +#ifndef _GNU_SOURCE + #define _GNU_SOURCE // for pthread_getname_np(), sync_file_range() +#endif // !_GNU_SOURCE + +#include "spew.h" +#include // for errno, EEXIST +#include // for sync_file_range() +#include // for backtrace, backtrace_symbols +#include // for pthread_mutex_lock, pthread_mutex_unlock +#include // for va_list, va_end, va_start +#include // for size_t, NULL, ptrdiff_t +#include // for uint64_t +#include // for fprintf, stderr, FILE, fputs, vsnprintf +#include // for abort, free +#include // for strnlen, strcmp, strncpy +#include // for stat, mkdir, S_IRWXG, S_IRWXO, S_IRWXU +#include // for CLOCK_REALTIME +#include // for tm, timespec, gmtime_r, clock_gettime +#include // for getpid +#include "backtrace.h" // for spew_backtrace +#include "tsan.h" // for NO_SANITIZE_THREAD +#include "../version.h" // for GITREV +#include "debug-tag.h" // for debt_snprintf +#include "simplest-defs.h" // for NELEM, STRINGIFY, boolint, STATIC_ASSERT +#include "sparse.h" // for __force +#include "error.h" +#include + + +#if defined(ASSERT) || defined(CHECK) +#error CANNOT_USE_ASSERT_OR_CHECK_IN_THIS_FILE +#endif + +struct spewlog { + // Everything in this struct is protected by this mutex. + // + pthread_mutex_t mutex; + + // Settings / flags. + // + size_t spew_level; + boolint suppress_spewage; + // Logs live in // + // --m--YY-mm-ddTHH-MM-00Z.log. + // + char logrootdir[512]; + char lfname_root[512]; + // One of: 1, 10, 60, 1440 (== 1, 10 minutes; 1, 24 hrs). + // A new roll period starts at midnight. + // + size_t roll_period_minutes; + boolint spew_to_stderr; + // Name and FILE* for file currently being spewed to. + // + // NOTE: This field is a sentinel whose existence serves to + // convince maybe_roll_logfile() not to continuously try to reopen + // a logfile in e.g. disk-full conditions. + // + char lfname[1024]; + FILE *logfile; +}; + +// Errors in the spew package can't call spew. By means of calling +// this method, spew-package error spewage goes to the logfile (if +// it's open), else stderr (which will fail if the process has already +// shut stderr, but in that case, there's nothing to do). +// +// This method may be called when log->mutex is not held -- from +// signal-handlers, for example. +// +static FILE * +spew_logfile(struct spewlog *log) +{ + FILE *logfile = log->logfile; + return logfile != NULL ? logfile : stderr; +} + +static void +spew_backtrace_simple_internal(FILE *f) +{ + void *trace[32]; + int n = backtrace(trace, (__force int)NELEM(trace)); + + if (n > 0) { + char **symbols = backtrace_symbols(trace, n); + + if (symbols) { + fprintf(f, "BACKTRACE:\n"); + for (size_t i = 0; + i < /*size_t_of_ptrdiff_t*/(__force size_t)(n); + i++) { + fprintf(f, " [%zu] %s\n", i, symbols[i]); + } + + free(symbols); + + } else { + perror("backtrace_symbols()"); + } + + } else { + perror("backtrace()"); + } +} + +void +spew_backtrace_simple(void) +{ + spew_backtrace_simple_internal(spew_logfile(&the_spew_log)); +} + +static void spew_backtrace_simple_and_abort(FILE *f) + __attribute__ ((__noreturn__)); +static void +spew_backtrace_simple_and_abort(FILE *f) +{ + spew_backtrace_simple_internal(f); + abort(); +} + + +/////////////////////////////////////////////////////////////////////////// +// Assertions in the spew package, which must not call log0() +// so as to avoid infinite recursion. +#define SPEW_CHECK(x) \ + ((x) ? (void) (0) : \ + spew_assert_fail(STRINGIFY(x), __FILE__, __LINE__, __PRETTY_FUNCTION__)) +#define SPEW_ASSERT SPEW_CHECK + +static void +spew_assert_fail(const char *assertion, + const char *file, size_t line, const char *function) +{ + FILE *f = spew_logfile(&the_spew_log); + fprintf(f, "FATAL: SPEW_ASSERTION_FAILURE: %s:%s:%s:%zu\n", + assertion, file, function, line); + spew_backtrace_simple_and_abort(f); +} + + +/////////////////////////////////////////////////////////////////////////// +// spew_level stuff +// + +static const char *spew_level_name[] = { + [SPEW_TRACE] = "TRACE", + [SPEW_DEBUG] = "DEBUG", + [SPEW_INFO] = "INFO", + [SPEW_TEST_INFO] = "TINFO", + [SPEW_WARN] = "WARN", + [SPEW_ERROR] = "ERROR", + [SPEW_BUG] = "BUG", + [SPEW_FATAL] = "FATAL", + [SPEW_CRIT] = "CRIT", +}; + +const char * +log_get_level_name(size_t level) +{ + SPEW_ASSERT(level < NELEM(spew_level_name)); + return spew_level_name[level]; +} + +size_t +log_level_of_level_name(const char *level_name) +{ + for (size_t level = FIRST_SPEW_LEVEL; level < N_SPEW_LEVELS; ++level) { + if (strcmp(level_name, spew_level_name[level]) == 0) { + return level; + } + } + fprintf(stderr, "FATAL: log_level_of_level_name(): " + "unrecognized level_name: '%s'\n", level_name); + spew_backtrace_simple_and_abort(stderr); +} + +static void +log_spew_level_change(size_t spew_level, size_t old_level, size_t new_level) +{ + SPEW_ASSERT(old_level < NELEM(spew_level_name)); + SPEW_ASSERT(new_level < NELEM(spew_level_name)); + spew(spew_level, "spew level change: %s -> %s", + spew_level_name[old_level], + spew_level_name[new_level]); +} + +size_t +log_set_level(struct spewlog *log, size_t new_level) +{ + SPEW_ASSERT(new_level < N_SPEW_LEVELS); + + // Spew the level change at the lower of the two levels; accomplished by + // spewing before the change if new > old, else after. + // + size_t old_level; + pthread_mutex_lock(&log->mutex); { + old_level = log->spew_level; + } pthread_mutex_unlock(&log->mutex); + if (new_level > old_level) { + log_spew_level_change(old_level, old_level, new_level); + } + pthread_mutex_lock(&log->mutex); { + old_level = log->spew_level; + log->spew_level = new_level; + } pthread_mutex_unlock(&log->mutex); + if (new_level <= old_level) { + log_spew_level_change(new_level, old_level, new_level); + } + + return old_level; +} + +/* spew_level += (add - sub), unsigned */ +size_t +log_add_level(struct spewlog *log, size_t add, size_t sub) +{ + SPEW_ASSERT(add == 0 || sub == 0); /* normal form */ + size_t level; + + /* This update is not atomic. There is no point in making it atomic + because log_set_level() is not atomic either. */ + pthread_mutex_lock(&log->mutex); { + level = log->spew_level; + } pthread_mutex_unlock(&log->mutex); + + level = (level + add >= N_SPEW_LEVELS) + ? N_SPEW_LEVELS - 1 + : level + add; + level = (level >= sub) + ? level - sub + : FIRST_SPEW_LEVEL; + return log_set_level(log, level); +} + +static pthread_mutex_t spew_level_lock = + PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP; +static uint64_t spew_level_counts[NELEM(spew_level_name)]; + +void +HACK_log_inc_level_count(size_t level) +{ + SPEW_ASSERT(level < NELEM(spew_level_counts)); + + pthread_mutex_lock(&spew_level_lock); { + spew_level_counts[level]++; + } pthread_mutex_unlock(&spew_level_lock); +} + +void +spew_reset_level_counts(uint64_t *level_counters, + size_t level_counters_len) +{ + SPEW_ASSERT(level_counters_len == NELEM(spew_level_counts)); + pthread_mutex_lock(&spew_level_lock); { + for (size_t level = 0; level < NELEM(spew_level_counts); ++level) { + level_counters[level] = spew_level_counts[level]; + spew_level_counts[level] = 0; + } + } pthread_mutex_unlock(&spew_level_lock); +} + + +/////////////////////////////////////////////////////////////////////////// +// logfile stuff +// + +// struct spewlog contains arrays with too many elements to initialize +// explicitly, so don't let sparse in on the secret. +// +struct spewlog the_spew_log +#ifndef __CHECKER__ += { + .mutex = PTHREAD_MUTEX_INITIALIZER, + .spew_level = SPEW_TEST_INFO, + .roll_period_minutes = SPEW_DEFAULT_ROLL_PERIOD, + .spew_to_stderr = 1, +} +#endif + ; +struct spewlog the_metrics_log +#ifndef __CHECKER__ += { + .mutex = PTHREAD_MUTEX_INITIALIZER, + .roll_period_minutes = SPEW_DEFAULT_ROLL_PERIOD, + .spew_to_stderr = 1, +} +#endif + ; + +static void +log_xstrncpy(char *dest, const char *src, size_t n) +{ + size_t len = strnlen(src, n); + SPEW_ASSERT(len < n); + // gcc 8.1 with -Werror=stringop-truncation doesn't like n as the argument + // to strncpy() since it figures out that n is exactly the size of dest. + strncpy(dest, src, n-1); + dest[len] = 0; +} + +static void +log_xbasename(char *dest, size_t dest_len, const char *path) +{ + // Copy path because some versions of basename() can modify their + // argument. + // + char path_copy[512]; + log_xstrncpy(path_copy, path, NELEM(path_copy)); + char *bn = basename(path_copy); + log_xstrncpy(dest, bn, dest_len); +} + +static int +log_xsnprintf(char *str, size_t size, const char *format, ...) + __attribute__((format(printf, 3, 4))); +static int +log_xsnprintf(char *str, size_t size, const char *format, ...) +{ + SPEW_ASSERT(str); + SPEW_ASSERT(format); + + va_list args; +#ifndef __CHECKER__ + // va_start() contains a cast to void * (from char const **, in + // this case), which runs afoul of sparse's check for qualifier + // drops. + // + va_start(args, format); +#endif // !__CHECKER__ + int size_otherwise = vsnprintf(str, size, format, args); + va_end(args); + SPEW_ASSERT(0 <= size_otherwise && + (__force size_t)(ptrdiff_t)size_otherwise < size); + return size_otherwise; +} + +static size_t +log_snprintf(char *str, size_t size, const char *format, ...) + __attribute__((format(printf, 3, 4))); +static size_t +log_snprintf(char *str, size_t size, const char *format, ...) +{ + va_list args; +#ifndef __CHECKER__ + // va_start() contains a cast to void * (from char const **, in + // this case), which runs afoul of sparse's check for qualifier + // drops. + // + va_start(args, format); +#endif // !__CHECKER__ + int size_otherwise = vsnprintf(str, size, format, args); + va_end (args); + SPEW_ASSERT(size_otherwise >= 0); + size_t so_pos = /*__force_size_t*/(__force size_t)(size_otherwise); + return (so_pos > size) ? size : so_pos; +} + +static size_t +log_vsnprintf(char *str, size_t size, const char *format, va_list args) + __attribute__((format(printf, 3, 0))); +static size_t +log_vsnprintf(char *str, size_t size, const char *format, va_list args) +{ + int size_otherwise = vsnprintf(str, size, format, args); + SPEW_ASSERT(size_otherwise >= 0); + size_t so_pos = /*__force_size_t*/(__force size_t)(size_otherwise); + return (so_pos > size) ? size : so_pos; +} + +static void +log_snprintf_logfile_tstr(char *tstr, size_t tstr_len, + const struct timespec *now) +{ + struct tm tm; + gmtime_r(&now->tv_sec, &tm); + + size_t len = strftime(tstr, tstr_len, "%Y-%m-%dT%H-%M-%S", &tm); + SPEW_ASSERT(len > 0); +} + +static void +log_snprintf_lfdir(struct spewlog *log, char *b, size_t b_len) +{ + log_xsnprintf(b, b_len, "%s/%s", log->logrootdir, log->lfname_root); +} + +static void +log_snprintf_lfname(struct spewlog *log, char *b, size_t b_len, + boolint include_sec_since_midnight_tag, + const struct timespec *now) +{ + struct tm tm; + gmtime_r(&now->tv_sec, &tm); + + // Construct "seconds since midnight" tag if needed. + // Note that it is necessary to do this before modifying tm. + // + char ssm_tag[16]; + if (include_sec_since_midnight_tag) { + int sec_since_midnight = + ((tm.tm_hour * 60) + tm.tm_min) * 60 + tm.tm_sec; + log_xsnprintf(ssm_tag, NELEM(ssm_tag), "%d.", sec_since_midnight); + } else { + ssm_tag[0] = '\0'; + } + + switch (log->roll_period_minutes) { + case 1440: + tm.tm_hour = 0; + /* fallthrough */ /* Make gcc7 happy with -Wimplicit-fallthrough */ + case 60: + tm.tm_min = 0; + /* fallthrough */ + case 10: + tm.tm_min = (tm.tm_min / 10) * 10; + case 1: + break; + default: { + FILE *f = spew_logfile(log); + fprintf(f, "FATAL: log_snprintf_lfname(): " + "unexpected log->roll_period_minutes=%zd\n", + log->roll_period_minutes); + spew_backtrace_simple_and_abort(f); + }} + + char lfdir[256]; + log_snprintf_lfdir(log, lfdir, NELEM(lfdir)); + + log_xsnprintf(b, b_len, + "%s/%s--%zdm--%04d-%02d-%02dT%02d-%02d-00Z.%slog", + lfdir, log->lfname_root, log->roll_period_minutes, + tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, + tm.tm_hour, tm.tm_min, ssm_tag); +} + +static void log_maybe_close_logfile(struct spewlog *log); + +/* + * Format and output a log line to the log file. + */ +static void +log_one_line(struct spewlog *log, size_t level, + const struct timespec *now, char const * function, + char const * filename, size_t line, char const * out_string) +{ + char l[1024]; + size_t li = 0; + + char now_str[64]; + log_snprintf_logfile_tstr(now_str, NELEM(now_str), now); + + char threadname[THREADNAME_SIZE]; + int ret = + pthread_getname_np(pthread_self(), threadname, sizeof(threadname)); + if (ret != 0) { + // If we cannot get the thread name just set it to + strncpy(threadname, "", NELEM(threadname)); + } + + char debt_buf[64]; + debt_snprintf(&debt_buf[0], sizeof(debt_buf)); + + char file_basename[512]; + log_xbasename(file_basename, NELEM(file_basename), filename); + + li += log_snprintf(l+li, NELEM(l)-li, + "%s.%06ldZ %d:%ld:%5s:%s:%s%s:%s:%zd: %s\n", + now_str, now->tv_nsec/1000, getpid(), + syscall(SYS_gettid), spew_level_name[level], + threadname, debt_buf, function, file_basename, line, + out_string); + + // Ensure that truncated log lines are newline-terminated. + STATIC_ASSERT(NELEM(l) > 2); + l[NELEM(l)-2] = '\n'; + l[NELEM(l)-1] = '\0'; + + // If logging has not been configured yet, spew to stderr. Once logging + // has been configured, when something goes wrong, logging gets diverted + // to stderr. When that is the case, only spew SPEW_ERROR and higher. + boolint early = (strnlen(log->lfname_root, NELEM(log->lfname_root)) == 0); + boolint spew_failed = !!0; + if (early || log->logfile != stderr || level >= SPEW_ERROR) { + spew_failed = fputs(l, log->logfile) == EOF; // EOF == any error. + } + + if (spew_failed || fflush(log->logfile) == EOF) { + fprintf(stderr, "ERROR: Write to '%s' failed (%s(%d)) -- " + "is filesystem full?\n", + log->lfname, xstrerror(errno), errno); + fprintf(stderr, "ERROR: all spewage now going to stderr\n"); + log_maybe_close_logfile(log); + log->logfile = stderr; + } + + if (log->logfile != stderr) { + int logfile_fd = fileno(log->logfile); + struct stat statbuf; + if (fstat(logfile_fd, &statbuf) != 0) { + fprintf(stderr, "ERROR: fstat('%s') failed (%s(%d))\n", + log->lfname, xstrerror(errno), errno); + } else if (sync_file_range( + logfile_fd, + 0, statbuf.st_size & ~(statbuf.st_blksize-1), + SYNC_FILE_RANGE_WRITE + ) != 0) { + fprintf(stderr, "ERROR: sync_file_range('%s') failed (%s(%d))\n", + log->lfname, xstrerror(errno), errno); + } + } + + // BUG and higher always goes to stderr as well. + if (1 + && (log->spew_to_stderr || level >= SPEW_BUG) + && spew_logfile(log) != stderr) { + fputs(l, stderr); + } +} + +/** + * Behaves like 'mkdir -p'. + * Copied (approximately) from util.c. + * If {,NON_}LEAF_MODE is 0, use rwxrwxrwx. + */ +static void +mkdir_p(const char *path, size_t path_len, mode_t leaf_mode, mode_t non_leaf_mode) +{ + char prefix[1024]; + size_t len = strnlen(path, path_len); + SPEW_ASSERT(len < NELEM(prefix)); + + mode_t rwxrwxrwx = (__force mode_t)(S_IRWXU|S_IRWXG|S_IRWXO); + if (leaf_mode == 0) { + leaf_mode = rwxrwxrwx; + } + if (non_leaf_mode == 0) { + non_leaf_mode = rwxrwxrwx; + } + + // For each successively larger prefix of pathname components (separated + // by '/' characters), if a directory of that name doesn't exist of that + // name, create it. + for (size_t i = 0; i < len + 1; i++) { + if (0 + || i == len + || (i > 0 && path[i] == '/')) { + + prefix[i] = 0; + + struct stat s; + if (stat(prefix, &s) == 0) { + // Directory exists. + SPEW_CHECK(S_ISDIR(s.st_mode)); + } else { + // Directory doesn't exist. Create it and set its mode bits. + + // Use leaf_mode when last/component or last/component/. + mode_t mode = i == len || (i == len-1 && path[i] == '/') ? leaf_mode : non_leaf_mode; + // Don't call spew() here (for obvious reasons). + fprintf(stderr, "INFO: mkdir(%s)\n", prefix); + int err = mkdir(prefix, mode); + if (err != 0 && err != EEXIST) { + // Skip EEXIST error -- this can happen in a race. + fprintf(stderr, "ERROR: mkdir(): errno = %s(%d)\n", + xstrerror(errno), errno); + } else { + // Call chmod() explicitly because mkdir() is modified by + // umask. + err = chmod(prefix, mode); + if (err != 0) { + fprintf(stderr, "ERROR: chmod(): errno = %s(%d)\n", + xstrerror(errno), errno); + } + } + } + } + + prefix[i] = path[i]; + } +} + +static void +log_maybe_close_logfile(struct spewlog *log) +{ + if (log->logfile != NULL && log->logfile != stderr) { + if (fsync(fileno(log->logfile)) != 0) { + fprintf(stderr, "ERROR: syncfs(\"%s\") failed: %s(%d)\n", + log->lfname, xstrerror(errno), errno); + } + if (fclose(log->logfile) != 0) { + fprintf(stderr, "ERROR: fclose(\"%s\") failed: %s(%d)\n", + log->lfname, xstrerror(errno), errno); + } + log->logfile = NULL; + // + // NOTE: Don't clear log->lfname here -- see comment for + // spewlog.lfname in structure declaration. + } +} + +void +log_config(struct spewlog *log, const char *logrootdir, const char *argv0, + size_t roll_period_m, boolint spew_to_stderr, boolint debug) +{ + // Ensure that spew_logfile() doesn't read garbage (it might be + // called by things we call). + // + log_maybe_close_logfile(log); + log->logfile = NULL; + + pthread_mutex_lock(&log->mutex); { + log_xstrncpy(log->logrootdir, logrootdir, NELEM(log->logrootdir)); + log_xbasename(log->lfname_root, NELEM(log->lfname_root), argv0); + + if (1 + && roll_period_m != 1 + && roll_period_m != 10 + && roll_period_m != 60 + && roll_period_m != 1440) { + fprintf(stderr, + "ERROR: unhandled roll_period_m: %zd; " + "using default (%d minutes)\n", + roll_period_m, SPEW_DEFAULT_ROLL_PERIOD); + roll_period_m = SPEW_DEFAULT_ROLL_PERIOD; + } + log->roll_period_minutes = roll_period_m; + log->spew_to_stderr = spew_to_stderr; + } pthread_mutex_unlock(&log->mutex); + + char lfdir[256]; + log_snprintf_lfdir(log, lfdir, NELEM(lfdir)); + mkdir_p(lfdir, NELEM(lfdir), + /*default leaf_mode: rwxrwxrwx*/0, /*default non_leaf_mode: rwxrwxrwx*/0); + + // HACK [dann 2017-02-13]: When configuring the non-metrics log, + // also configure the metrics log for this process -- use the same + // parameters for the metrics log but a different + // log-directory/logfile-name, and set the level differently as + // well. + // + if (log == &the_metrics_log) { + log_set_level(log, 0); + } else { + if (debug) { + log_set_level(log, SPEW_DEBUG); + } else { + log_set_level(log, SPEW_INFO); + } + + char new_argv0[256]; + log_xsnprintf(new_argv0, NELEM(new_argv0), "%s-mlog", argv0); + log_config(&the_metrics_log, logrootdir, new_argv0, roll_period_m, + /*spew_to_stderr=*/!!0, /*debug=*/!!0); + } + +} + +void +log_teardown(struct spewlog *log) +{ + pthread_mutex_lock(&log->mutex); { + log_maybe_close_logfile(log); + log->logrootdir[0] = '\0'; + log->lfname_root[0] = '\0'; + } pthread_mutex_unlock(&log->mutex); +} + +static /*reopen*/boolint +rename_logfile_if_too_large(struct spewlog *log, const struct timespec *now) +{ + SPEW_ASSERT(log->lfname[0] != '\0'); + + struct stat s; + if (0 + || log->logfile == NULL + || log->logfile == stderr + || stat(log->lfname, &s) != 0) + { + // Don't rename logfile if not spewing to it or if we can't + // stat it (for whatever reason). + // + return 0; + } + + if (s.st_size > SPEW_MAX_LOGFILE_SIZE) { + char alt_lfname[NELEM(log->lfname)]; + log_snprintf_lfname(log, alt_lfname, NELEM(alt_lfname), + /*include_sec_since_midnight_tag=*/!0, now); + fprintf(stderr, "renaming '%s' ==> '%s'\n", log->lfname, alt_lfname); + if (rename(log->lfname, alt_lfname) == 0) { + // Ensure that roll_logfile will reopen a new logfile, + // now that the old one has been moved out of the way. + // + log->lfname[0] = '\0'; + return !0; + } + fprintf(stderr, "ERROR: rename('%s' ==> '%s') failed: %s(%d)\n", + log->lfname, alt_lfname, xstrerror(errno), errno); + log_maybe_close_logfile(log); + log->logfile = stderr; + fprintf(stderr, "ERROR: all spewage now going to stderr\n"); + } + return 0; +} + +// Open a new logfile, or close the currently-open file and open a new +// one if it's time to roll, or if the current logfile is too large +// for lumberjack's liking. If the open fails, or if log_config() has +// not been called yet, use stderr. +// +static void +maybe_roll_logfile(struct spewlog *log, const struct timespec *now) +{ + // Don't try to open if spew_config() hasn't been called yet. + // + if (strnlen(log->lfname_root, NELEM(log->lfname_root)) == 0) { + log->logfile = stderr; + return; + } + + char desired_lfname[256]; + log_snprintf_lfname(log, desired_lfname, NELEM(desired_lfname), + /*include_sec_since_midnight_tag=*/0, now); + boolint should_roll = 0 + || strcmp(desired_lfname, log->lfname) != 0 + || rename_logfile_if_too_large(log, now); + + if (should_roll) { + log_maybe_close_logfile(log); + SPEW_ASSERT(log->logfile == NULL || log->logfile == stderr); + } + + if (log->logfile == NULL || (log->logfile == stderr && should_roll)) { + SPEW_ASSERT(strnlen(desired_lfname, NELEM(desired_lfname)) < + NELEM(log->lfname)); + log_xstrncpy(log->lfname, desired_lfname, NELEM(log->lfname)-1); + log->logfile = fopen(desired_lfname, "ae"); + if (log->logfile == NULL) { + fprintf(stderr, "ERROR: failed to open '%s' (%s(%d))\n", + desired_lfname, xstrerror(errno), errno); + fprintf(stderr, "ERROR: all spewage now going to stderr\n"); + log->logfile = stderr; + } else { + /* Logfile rolled. Set the first line to the gitrev */ + if (log != &the_metrics_log) { + log_one_line(log, SPEW_INFO, now, __FUNCTION__, + __FILE__, (size_t)__LINE__, gitrev); + } + } + } + + SPEW_ASSERT(log->logfile != NULL); + + /* set LOGFILE to be line-buffered */ + setvbuf(log->logfile, NULL, _IOLBF, 0); +} + + +/////////////////////////////////////////////////////////////////////////// +// spew() routines. +// + +// Helper routine for log{0,1}(). +// +static void +log_gettime_or_abort(struct spewlog *log, struct timespec *now) +{ + if (clock_gettime(CLOCK_REALTIME, now) != 0) { + FILE *f = spew_logfile(log); + fprintf(f, + "FATAL: clock_gettime() failed: %s(%d)\n", + xstrerror(errno), errno); + spew_backtrace_simple_and_abort(f); + } +} + +boolint +log_enabled(struct spewlog *log, size_t level) + NO_SANITIZE_THREAD /* we don't acquire the lock to read log->spew_level. + Not worth it. */ + +{ + return (level >= log->spew_level); +} + +// logv() is the main mechanism for writing to an application log file. +// +// DO NOT CALL CHECK()/ASSERT() FROM THIS FUNCTION OR ANYTHING IT +// CALLS OR ELSE SUFFER INFINITE RECURSION !!! +// +void +logv(struct spewlog *log, + boolint honor_spewage_suppression, + boolint maybe_roll, + boolint is_stack_trace, + size_t level, + const char *file, + size_t line, + const char *fn, + const char *fmt, + va_list ap) +{ + if (level < log->spew_level) { + return; + } + + HACK_log_inc_level_count(level); + boolint did_spew; + + pthread_mutex_lock(&log->mutex); { + + if (honor_spewage_suppression && log->suppress_spewage) { + // Suppress spewage. + did_spew = 0; + } else { + struct timespec now; + log_gettime_or_abort(log, &now); + if (maybe_roll || log->logfile == NULL) { + maybe_roll_logfile(log, &now); + } + + char l[1024]; + size_t ls = NELEM(l); + log_vsnprintf(l, ls, fmt, ap); + l[ls-1] = '\0'; + + log_one_line(log, level, &now, fn, file, line, l); + did_spew = 1; + } + + } pthread_mutex_unlock(&log->mutex); + + // Add backtrace to BUG spewage (which is not fatal, so we we wouldn't + // otherwise see a backtrace). + if (did_spew && level == SPEW_BUG && !is_stack_trace) { + spew_backtrace(SPEW_BUG); + } +} + + +// log0(), wrapped by spew() in util.h, is the main mechanism for +// writing to an application log file. +// +// DO NOT CALL CHECK()/ASSERT() FROM THIS FUNCTION OR ANYTHING IT +// CALLS OR ELSE SUFFER INFINITE RECURSION !!! +// +void +log0(struct spewlog *log, + boolint honor_spewage_suppression, + boolint maybe_roll, + boolint is_stack_trace, + size_t level, + const char *file, + size_t line, + const char *fn, + const char *fmt, + ...) +{ + va_list ap; +#ifndef __CHECKER__ + // va_start() contains a cast to void * (from char const **, in this + // case), which runs afoul of sparse's check for qualifier drops. + // + va_start(ap, fmt); +#endif // !__CHECKER__ + logv(log, honor_spewage_suppression, maybe_roll, is_stack_trace, + level, file, line, fn, fmt, ap); + va_end(ap); +} + +void +log1(struct spewlog *log, + const char *fmt, + ...) +{ + SPEW_ASSERT(log->spew_level == 0); // UNUSED + + va_list ap; + + pthread_mutex_lock(&log->mutex); { + + struct timespec now; + log_gettime_or_abort(log, &now); + maybe_roll_logfile(log, &now); + + if (log->logfile != stderr) { + char now_str[64]; + log_snprintf_logfile_tstr(now_str, NELEM(now_str), &now); + + char l[1024]; + size_t ls = NELEM(l); + size_t li = 0; + + li += log_snprintf(l+li, ls-li, + "%s.%06ldZ: ", now_str, now.tv_nsec/1000); + +#ifndef __CHECKER__ + // va_start() contains a cast to void * (from char const **, + // in this case), which runs afoul of sparse's check for + // qualifier drops. + // + va_start(ap, fmt); +#endif // !__CHECKER__ + li += log_vsnprintf(l+li, ls-li, fmt, ap); + va_end(ap); + li += log_snprintf(l+li, ls-li, "\n"); + + // Ensure that truncated log lines are newline-terminated. + // + l[ls-2] = '\n'; + l[ls-1] = '\0'; + + if (fputs(l, log->logfile) == EOF) { // EOF == any error. + fprintf(stderr, "ERROR: Write to '%s' failed (%s(%d)) -- " + "is filesystem full?\n", + log->lfname, xstrerror(errno), errno); + log_maybe_close_logfile(log); + log->logfile = stderr; + fprintf(stderr, "ERROR: no longer spewing metrics to %s\n", + log->lfname); + } + } + } pthread_mutex_unlock(&log->mutex); +} + +boolint +spew_logfile_ok(struct spewlog *log) +{ + boolint ok; + pthread_mutex_lock(&log->mutex); { + struct timespec now; + log_gettime_or_abort(log, &now); + maybe_roll_logfile(log, &now); + ok = (log->logfile != stderr); + } pthread_mutex_unlock(&log->mutex); + return ok; +} + +void +spew_config(int argc, char *const argv[], const char *logrootdir, + size_t roll_period_m, boolint spew_to_stderr, boolint debug) +{ + SPEW_ASSERT(argc > 0); + log_config(&the_spew_log, logrootdir, argv[0], + roll_period_m, spew_to_stderr, debug); + spew(SPEW_INFO, "============= STARTING ============="); + spew(SPEW_INFO, "GITREV = %s", gitrev); + spew(SPEW_INFO, "ARGV = ["); + for (size_t i = 0; i < (__force size_t)argc; i++) { + spew(SPEW_INFO, " \"%s\",", argv[i]); + } + spew(SPEW_INFO, "log directory: %s", logrootdir); + spew(SPEW_INFO, "]"); +} + +void +spew_teardown(void) +{ + spew(SPEW_INFO, "goodbye"); + log_teardown(&the_spew_log); + log_teardown(&the_metrics_log); +} + + +size_t +spew_get_level(void) { + // Different from DEBUG_spew_get_level in that it acquires a lock and can be used. + size_t level; + pthread_mutex_lock(&the_spew_log.mutex); { + level = the_spew_log.spew_level; + } pthread_mutex_unlock(&the_spew_log.mutex); + return level; +} + +/////////////////////////////////////////////////////////////////////////// +// DEBUG +// + +boolint +DEBUG_shutup(boolint suppress) +{ + boolint old; + + pthread_mutex_lock(&the_spew_log.mutex); { + old = the_spew_log.suppress_spewage; + the_spew_log.suppress_spewage = suppress; + } pthread_mutex_unlock(&the_spew_log.mutex); + + return old; +} + +size_t +DEBUG_spew_get_level(void) +{ + return the_spew_log.spew_level; +} + + +#if 0 +/////////////////////////////////////////////////////////////////////////// +// leak-module link hack +// + +/* hack to link the leak checker, so that we can enforce + statically that the leak checker does not call spew() */ +extern void (*META_HACK_force_linking_of_leak_module)(void); +void (*META_HACK_force_linking_of_leak_module)(void) = + HACK_force_linking_of_leak_module; +#endif diff --git a/software/fss-parallel-tools/common/spew.h b/software/fss-parallel-tools/common/spew.h new file mode 100644 index 0000000..a2c76e3 --- /dev/null +++ b/software/fss-parallel-tools/common/spew.h @@ -0,0 +1,175 @@ +#ifndef __SPEW_H__ +#define __SPEW_H__ + + +#include // for va_list +#include // for size_t +#include // for uint64_t +#include "simplest-defs.h" // for boolint +struct spewlog; // lines 13-13 + + +enum { + SPEW_DEFAULT_ROLL_PERIOD = 1440, // go: logging.DefaultLogfileRollPeriod + // go: logging.MaxLogFileSize + SPEW_MAX_LOGFILE_SIZE = 1024ll * 1024ll * 512ll, + THREADNAME_SIZE = 16, // <== man pthread_setname_np() +}; + +/* Two INFO levels. + + The theory is that in production we want INFO about + open/close/dropped connections, how many extents we found on disk, + etc. Such INFO is kind-of useless and distracting for tests. + Thus, the default verbosity is SPEW_TEST_INFO for tests. + Production code should run at SPEW_INFO and print more spewage. +*/ +enum { + FIRST_SPEW_LEVEL = 0u, + SPEW_TRACE = 0u, + SPEW_DEBUG = 1u, + SPEW_INFO = 2u, + SPEW_TEST_INFO = 3u, + SPEW_WARN = 4u, + SPEW_ERROR = 5u, + SPEW_BUG = 6u, + SPEW_FATAL = 7u, + SPEW_CRIT = 8u, + N_SPEW_LEVELS = 9u, +}; + + +extern struct spewlog the_spew_log; +extern struct spewlog the_metrics_log; + + +const char * +log_get_level_name(size_t level); +size_t +log_level_of_level_name(const char *level_name); +size_t +log_set_level(struct spewlog *log, size_t new_level); +size_t +log_add_level(struct spewlog *log, size_t add, size_t sub); + +void +HACK_log_inc_level_count(size_t level); +void +spew_reset_level_counts(uint64_t *level_counters, size_t level_counters_len); + +void +log_config(struct spewlog *log, const char *logrootdir, const char *argv0, + size_t roll_period_m, boolint spew_to_stderr, boolint debug); +void log_teardown(struct spewlog *log); + +boolint +log_enabled(struct spewlog *log, size_t level); + +void +logv(struct spewlog *log, + boolint honor_spewage_suppression, + boolint maybe_roll, + boolint is_stack_trace, + size_t level, + const char *file, + size_t line, + const char *fn, + const char *fmt, + va_list ap) + __attribute__((format(printf, 9, 0))); + +void log0(struct spewlog *log, + boolint honor_spewage_suppression, + boolint maybe_roll, + boolint is_stack_trace, + size_t level, + const char *file, + size_t line, + const char *fn, + const char *fmt, + ...) + __attribute__ ((format (printf, 9, 10))); +void +log1(struct spewlog *log, + const char *fmt, + ...) + __attribute__ ((format (printf, 2, 3))); +boolint +spew_logfile_ok(struct spewlog *log); + +/* Legacy APIs. + */ +static inline const char * +spew_get_level_name(size_t level) +{ + return log_get_level_name(level); +} +static inline size_t +spew_set_level(size_t new_level) +{ + return log_set_level(&the_spew_log, new_level); +} +static inline size_t spew_more(void) +{ + return log_add_level(&the_spew_log, /*add=*/0, /*sub=*/1); +} +static inline size_t spew_less(void) +{ + return log_add_level(&the_spew_log, /*add=*/1, /*sub=*/0); +} + +/* Simple backtrace facility. + */ +void spew_backtrace_simple(void); + +#define would_spew(lvl) log_enabled(&the_spew_log, (lvl)) + +#define spew(lvl, fmt, ...) \ + ({ \ + if (log_enabled(&the_spew_log, lvl)) { \ + log0(&the_spew_log, \ + /*honor_spewage_suppression=*/1, /*maybe_roll=*/1, \ + /*is_stack_trace=*/0, \ + lvl, __FILE__, __LINE__, __FUNCTION__, \ + fmt, ##__VA_ARGS__); \ + } \ + }) + +#define spew_no_roll(lvl, fmt, ...) \ + ({ \ + if (log_enabled(&the_spew_log, lvl)) { \ + log0(&the_spew_log, \ + /*honor_spewage_suppression=*/1, /*maybe_roll=*/0, \ + /*is_stack_trace=*/0, \ + lvl, __FILE__, __LINE__, __FUNCTION__, \ + fmt, ##__VA_ARGS__); \ + } \ + }) + +#define spew_for_backtrace(lvl, file, line, function, symbol) \ + ({ \ + if (log_enabled(&the_spew_log, lvl)) { \ + log0(&the_spew_log, \ + /*honor_spewage_suppression=*/1, /*maybe_roll=*/0, \ + /*is_stack_trace=*/1, \ + lvl, file, line, function, "%s", symbol); \ + } \ + }) + +void +spew_config(int argc, char *const argv[], const char *logrootdir, + size_t roll_period_m, boolint spew_to_stderr, boolint debug); +void +spew_teardown(void); + +size_t +spew_get_level(void); +// Different from DEBUG_spew_get_level because it acquires a lock. + +boolint +DEBUG_shutup(boolint suppress); +size_t +DEBUG_spew_get_level(void); + + +#endif // !__SPEW_H__ diff --git a/software/fss-parallel-tools/common/tsan.h b/software/fss-parallel-tools/common/tsan.h new file mode 100644 index 0000000..bf9dcd8 --- /dev/null +++ b/software/fss-parallel-tools/common/tsan.h @@ -0,0 +1,15 @@ +#ifndef __TSAN_H__ +#define __TSAN_H__ + +/* NO_SANITIZE_THREAD: disable TSAN on a function */ +#ifdef __clang__ +# if __has_attribute(__no_sanitize__) +# define NO_SANITIZE_THREAD __attribute__((no_sanitize("thread"))) +# endif +#endif + +#ifndef NO_SANITIZE_THREAD +#define NO_SANITIZE_THREAD /**/ +#endif + +#endif diff --git a/software/fss-parallel-tools/fdleak.c b/software/fss-parallel-tools/fdleak.c new file mode 100644 index 0000000..091ffa6 --- /dev/null +++ b/software/fss-parallel-tools/fdleak.c @@ -0,0 +1,64 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "fdleak.h" +#include +#include +#include +#include // for getrlimit, rlimit, RLIMIT_NO... + +// File descriptor leak check +static size_t n_open_fds_at_start = 0; +enum { OFAS_SIZE = 100 }; +enum { CHECK_FD_LIMIT = 1024 }; +static int open_fds_at_start[OFAS_SIZE]; +static int fd_is_open(int fd) { + // Use fcntl F_GETFL to figure out if file is open + return fcntl(fd, F_GETFL) != -1; +} +void find_open_fds_at_start(void) { + struct rlimit rlim; + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { + fprintf(stderr, "FATAL: getrlimit() failed\n"); + abort(); + } + for (int fd = 0; fd < (int)rlim.rlim_cur && fd < CHECK_FD_LIMIT; ++fd) { + if (fd_is_open(fd)) { + if (n_open_fds_at_start < OFAS_SIZE) { + open_fds_at_start[n_open_fds_at_start++] = fd; + } else { + fprintf(stderr, "FATAL: too many open_fds_at_start %zd\n", + n_open_fds_at_start); + abort(); + } + } + } +} + +static int fd_was_open(int fd) { + for (size_t i = 0; i < n_open_fds_at_start; ++i) { + if (fd == open_fds_at_start[i]) { + return !!1; + } + } + return !!0; +} + +void fdleak_check(void) +{ + struct rlimit rlim; + if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) { + fprintf(stderr, "FATAL: getrlimit() failed\n"); + abort(); + } + + for (int fd = 0; fd < (int)rlim.rlim_cur && fd < CHECK_FD_LIMIT; ++fd) { + if (!fd_was_open(fd) && fd_is_open(fd)) { + fprintf(stderr, "BUG: fd %d still open at end of program\n", fd); + } + } +} diff --git a/software/fss-parallel-tools/fdleak.h b/software/fss-parallel-tools/fdleak.h new file mode 100644 index 0000000..e3494ee --- /dev/null +++ b/software/fss-parallel-tools/fdleak.h @@ -0,0 +1,20 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef FDLEAK_H +#define FDLEAK_H + +void find_open_fds_at_start(void); +// Effect: Make a note of all the open file descriptors. Typically +// you call this first thing. + +void fdleak_check(void); +// Effect: Emit an error diagnostic if any file descriptors are open +// that weren't open when find_open_fds_at_start() ran. Typically you +// call this at the end. + +#endif diff --git a/software/fss-parallel-tools/fwg.c b/software/fss-parallel-tools/fwg.c new file mode 100644 index 0000000..3ab96a6 --- /dev/null +++ b/software/fss-parallel-tools/fwg.c @@ -0,0 +1,373 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "fwg.h" +#include "malloc.h" +#include "hashfun.h" +#include +#include +#include +#include + +struct nodebag { + size_t n; + size_t size; + struct fwgnode **nodes; +}; + +static struct nodebag *mk_nodebag() { + const size_t initial_size = 2; + struct nodebag *MALLOC(result); + struct fwgnode **MALLOC_N(nodes, initial_size); + *result = (struct nodebag){0, initial_size, nodes}; + return result; +} +static struct nodebag *nodebag_destroy(struct nodebag *nodebag) { + assert(nodebag->n == 0); + FREE(nodebag->nodes); + FREE(nodebag); + return NULL; +} +static void nodebag_push(struct nodebag *nodebag, struct fwgnode *node) { + if (nodebag->n * 2 >= nodebag->size) { + nodebag->size *= 2; + REALLOC(nodebag->nodes, nodebag->size); + } + assert(nodebag->n * 2 < nodebag->size); + nodebag->nodes[nodebag->n++] = node; +} +static int nodebag_is_empty(struct nodebag *nodebag) { + return nodebag->n == 0; +} +static int nodebag_pop_random(struct nodebag *nodebag, struct fwgnode **resultp) { + if (nodebag_is_empty(nodebag)) return 1; + size_t randv = (size_t)(random()) % nodebag->n; + struct fwgnode *result = nodebag->nodes[randv]; + nodebag->nodes[randv] = nodebag->nodes[--nodebag->n]; + *resultp = result; + return 0; +} +static int nodebag_pop_arbitrary(struct nodebag *nodebag, struct fwgnode **result) { + if (nodebag_is_empty(nodebag)) return 1; + *result = nodebag->nodes[--nodebag->n]; + return 0; +} + +struct fwgnode { + size_t ready_count; // how many predecessors are + // unfinished. + size_t cost; + char *name; + struct fwobject *fwobject; + struct nodebag *successors; +}; + +struct nodetable { + size_t n; // how many non-null items in nodes. + size_t size; // a power of two. + struct fwgnode **entries; // an array of length size. +}; + +static void nodetable_verify(const struct nodetable *n) { + assert(0 == (n->size & (n->size - 1))); + assert(n->n <= n->size); + if (0) { + size_t count = 0; + for (size_t i = 0; i < n->size; i++) { + if (n->entries[i]) count++; + } + assert(count == n->n); + } +} + +static struct nodetable *mk_nodetable(void) { + struct nodetable *MALLOC(result); + const size_t initsize = 2; + struct fwgnode **MALLOC_N(nodeps, initsize); + for (size_t i = 0; i < initsize; i++) { + nodeps[i] = NULL; + } + *result = (struct nodetable){0, initsize, nodeps}; + nodetable_verify(result); + return result; +} +static struct nodetable *nodetable_destroy(struct nodetable *nodetable) { + nodetable_verify(nodetable); + assert(nodetable->n == 0); + FREE(nodetable->entries); + FREE(nodetable); + return NULL; +} +static int nodetable_is_empty(struct nodetable *nodetable) { + return nodetable->n == 0; +} +static void nodetable_insert(struct nodetable *nodetable, + struct fwgnode *node); +static void nodetable_rehash(struct nodetable *nodetable, size_t new_size) { + const size_t old_size = nodetable->size; + struct fwgnode **old_entries = nodetable->entries; + MALLOC_N(nodetable->entries, new_size); + for (size_t i = 0; i < new_size; i++) { + nodetable->entries[i] = NULL; + } + nodetable->n = 0; + nodetable->size = new_size; + for (size_t i = 0; i < old_size; i++) { + if (old_entries[i]) { + nodetable_insert(nodetable, old_entries[i]); + } + } + FREE(old_entries); +} + +static size_t nodetable_find_index(const struct nodetable *nodetable, + const char *name) { + assert(0 == (nodetable->size & (nodetable->size - 1))); // power of 2 + size_t hstart = hash_string(name); + for (size_t count = 0; count < nodetable->size; count++) { + size_t hi = (hstart+count) & (nodetable->size -1); + if (nodetable->entries[hi] == NULL) return hi; + if (strcmp(nodetable->entries[hi]->name, name) == 0) { + return hi; + } + } + assert(0); // unreachable +} +static void nodetable_insert(struct nodetable *nodetable, + struct fwgnode *node) { + if (0) printf("Inserting %s, n=%lu size=%lu\n", node->name, nodetable->n, nodetable->size); + nodetable_verify(nodetable); + if (0) printf("Verified\n"); + if (nodetable->n * 2 >= nodetable->size) { + if (0) printf("rehashing\n"); + nodetable_rehash(nodetable, nodetable->size * 2); + } + nodetable_verify(nodetable); + assert(nodetable->n * 2 < nodetable->size); + size_t hi = nodetable_find_index(nodetable, node->name); + assert(nodetable->entries[hi] == NULL); + nodetable->entries[hi] = node; + nodetable->n++; + if (0) printf("%s:%d n=%lu\n", __FILE__, __LINE__, nodetable->n); + nodetable_verify(nodetable); + if (0) printf("%s:%d ok\n", __FILE__, __LINE__); +} +static void nodetable_reinsert_from(struct nodetable *nodetable, + const size_t start) { + for (size_t count = 0; count < nodetable->size; count++) { + size_t hi = (start + count) & (nodetable->size - 1); + struct fwgnode *node = nodetable->entries[hi]; + if (node == NULL) return; + nodetable->entries[hi] = NULL; + nodetable->n--; + nodetable_insert(nodetable, node); + } +} +static void nodetable_remove(struct nodetable *nodetable, + struct fwgnode *node) { + if (0) printf("%s:%d removing\n", __FILE__, __LINE__); + nodetable_verify(nodetable); + size_t hi = nodetable_find_index(nodetable, node->name); + assert(nodetable->entries[hi] == node); + nodetable->n--; + nodetable->entries[hi] = NULL; + nodetable_reinsert_from(nodetable, hi+1); + if (0) printf("%s:%d removed\n", __FILE__, __LINE__); + nodetable_verify(nodetable); +} +static int nodetable_find(struct nodetable *nodetable, + const char *name, + struct fwgnode **result) { + size_t hi = nodetable_find_index(nodetable, name); + if (nodetable->entries[hi] == NULL) return 1; + assert(0 == strcmp(nodetable->entries[hi]->name, name)); + *result = nodetable->entries[hi]; + return 0; +} + +struct fwg { + pthread_mutex_t mutex; + pthread_cond_t wait_for_ready; + pthread_cond_t wait_for_notfull; + int done; + size_t budget; + size_t spent; + struct nodebag *ready_nodes; + struct nodetable *nodes; // A map from names to nodes +}; +struct fwg_nodehandle; +struct fwobject; /* Opaque */ + +static void fwg_verify(const struct fwg *fwg) { + nodetable_verify(fwg->nodes); +} + +struct fwg *mk_fwg(size_t budget) { + struct fwg *MALLOC(result); + pthread_mutex_init(&result->mutex, NULL); + pthread_cond_init(&result->wait_for_ready, NULL); + pthread_cond_init(&result->wait_for_notfull, NULL); + result->done = 0; + result->budget = budget; + result->spent = 0; + result->ready_nodes = mk_nodebag(); + result->nodes = mk_nodetable(); + fwg_verify(result); + return result; +} + +struct fwg *fwg_destroy(struct fwg *fwg) { + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + pthread_mutex_destroy(&fwg->mutex); + pthread_cond_destroy(&fwg->wait_for_ready); + pthread_cond_destroy(&fwg->wait_for_notfull); + assert(nodebag_is_empty(fwg->ready_nodes)); + fwg->ready_nodes = nodebag_destroy(fwg->ready_nodes); + if (!nodetable_is_empty(fwg->nodes)) { + printf("Nodetable->n=%ld\n", fwg->nodes->n); + for (size_t i = 0; i < fwg->nodes->size; i++ ){ + if (fwg->nodes->entries[i]) { + printf(" %p: %s:\n", fwg->nodes->entries[i], fwg->nodes->entries[i]->name); + } + } + } + assert(nodetable_is_empty(fwg->nodes)); + fwg->nodes = nodetable_destroy(fwg->nodes); + FREE(fwg); + return NULL; +} + +static void add_dependency(struct fwg *fwg, + const char *prevname, + struct fwgnode *node) { + // NULL or empty previous is not a dependency + if (prevname==NULL || 0 == *prevname) return; + struct fwgnode *prevnode; + if (0) printf("adding dependency from %s to %p\n", prevname, node); + if (nodetable_find(fwg->nodes, prevname, &prevnode) == 0) { + if (0) printf(" Found: adding dependency from %p to %p\n", prevnode, node); + nodebag_push(prevnode->successors, node); + node->ready_count++; + if (0) printf(" Found prev node (ready_count=%lu)\n", node->ready_count); + } else { + if (0) printf(" No prev node (ready_count=%lu)\n", node->ready_count); + } +} + +static void note_maybe_node_is_ready(struct fwg *fwg, + struct fwgnode *node) { + if (0) printf("ready_count=%lu\n", node->ready_count); + if (node->ready_count == 0) { + nodebag_push(fwg->ready_nodes, node); + pthread_cond_signal(&fwg->wait_for_ready); + if (0) printf("%s:%d n ready = %lu\n", __FILE__, __LINE__, fwg->ready_nodes->n); + } +} + +static void fwg_add_internal(struct fwg *fwg, + const char *name, + struct fwobject *fwobject, + size_t cost, + const char *prev1, + const char *prev2, + const char *prev3) { + struct fwgnode *MALLOC(node); + if (0) printf("Add node %p %s %s %s %s\n", node, name, prev1, prev2, prev3); + pthread_mutex_lock(&fwg->mutex); + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + *node = (struct fwgnode){0, cost, strdup(name), fwobject, mk_nodebag()}; + add_dependency(fwg, name, node); + add_dependency(fwg, prev1, node); + add_dependency(fwg, prev2, node); + add_dependency(fwg, prev3, node); + if (0) printf("%s:%d Inserting\n", __FILE__, __LINE__); + nodetable_insert(fwg->nodes, node); + if (0) printf("%s:%d Inserted\n", __FILE__, __LINE__); + note_maybe_node_is_ready(fwg, node); + fwg->spent += cost; + while (fwg->spent > fwg->budget) { + pthread_cond_wait(&fwg->wait_for_notfull, &fwg->mutex); + } + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + pthread_mutex_unlock(&fwg->mutex); +} +void fwg_add1(struct fwg *fwg, + const char *name, struct fwobject *fwobject, size_t budget, + const char *prev1) { + fwg_add_internal(fwg, name, fwobject, budget, prev1, NULL, NULL); +} + +void fwg_add2(struct fwg *fwg, + const char *name, struct fwobject *fwobject, size_t budget, + const char *prev1, const char *prev2) { + fwg_add_internal(fwg, name, fwobject, budget, prev1, prev2, NULL); +} +void fwg_add3(struct fwg *fwg, + const char *name, struct fwobject *fwobject, size_t budget, + const char *prev1, const char *prev2, const char *prev3) { + fwg_add_internal(fwg, name, fwobject, budget, prev1, prev2, prev3); +} +void fwg_end_of_nodes(struct fwg *fwg) { + if (0) printf("End of nodes\n"); + pthread_mutex_lock(&fwg->mutex); + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + assert(!fwg->done); + fwg->done = 1; + pthread_cond_broadcast(&fwg->wait_for_ready); + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + pthread_mutex_unlock(&fwg->mutex); +} +int fwg_get_ready_node(struct fwg *fwg, + struct fwobject **fwobject, + struct fwgnode **handle) { + pthread_mutex_lock(&fwg->mutex); + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + while (!fwg->done && nodebag_is_empty(fwg->ready_nodes)) { + pthread_cond_wait(&fwg->wait_for_ready, &fwg->mutex); + if (0) printf("%s:%d awoke\n", __FILE__, __LINE__); + } + struct fwgnode *result; + int r = nodebag_pop_random(fwg->ready_nodes, &result); + //printf("r=%d\n", r); + if (r != 0) { + assert(fwg->done); + } else { + assert(result->cost <= fwg->spent); + if (fwg->spent > fwg->budget + && fwg->spent - result->cost <= fwg->budget) + { + pthread_cond_broadcast(&fwg->wait_for_notfull); + } + fwg->spent -= result->cost; + *fwobject = result->fwobject; + *handle = result; + } + //printf("%s:%d\n", __FILE__, __LINE__); fwg_verify(fwg); + pthread_mutex_unlock(&fwg->mutex); + if (0) printf("%s:%d returning %d\n", __FILE__, __LINE__, r); + return r; +} + +void fwg_finish_node(struct fwg *fwg, struct fwgnode *handle) { + pthread_mutex_lock(&fwg->mutex); + if (0) printf("Finishing %p which has %lu successors\n", handle, handle->successors->n); + while (1) { + struct fwgnode *succ; + int r = nodebag_pop_arbitrary(handle->successors, &succ); + if (r != 0) break; + if (0) printf(" Maybe wake up %p (pred=%lu)\n", succ, succ->ready_count); + assert(succ->ready_count > 0); + succ->ready_count--; + note_maybe_node_is_ready(fwg, succ); + if (0) printf(" succ->ready_count=%lu\n", succ->ready_count); + } + nodetable_remove(fwg->nodes, handle); + handle->successors = nodebag_destroy(handle->successors); + FREE(handle->name); + FREE(handle); + pthread_mutex_unlock(&fwg->mutex); +} diff --git a/software/fss-parallel-tools/fwg.h b/software/fss-parallel-tools/fwg.h new file mode 100644 index 0000000..64d32eb --- /dev/null +++ b/software/fss-parallel-tools/fwg.h @@ -0,0 +1,78 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef FWG_H +#define FWG_H +/* File work graph. */ + +/* A file work graph is a directed acyclic graph (DAG) of nodes, each + * containing an fwobject. + + * This structure is multithreaded, with some threads adding nodes, + * and some removing nodes that have no predecessors. + * + * The graph has a budget, measured in bytes. The graph avoids + * allocating more memory than the budget permits, so if you try to + * add too many nodes, you will stall until some other thread removes + * objects. + + * A fwobject is opaque from this module's perspective. + * + * fwobjects are identified by a pathname (a string). We assume that + * symlinks aren't being used to confuse us by using different + * pathnames for the same directory. + * + * To create an object (a file, hardlink, symlink, or directory, for + * example), the directory of the object must have been created, and + * the previous object in the same directory must have been created. + * For hardlinks, the linked-to object must also have been created. + * + * To make this work we keep a hash table that maps from pathnames to nodes. + * + * Pathnames for regular files point at the node that creates the + * file. When the file is created, we remove the hashtable entry. + * + * Pathnames for directories point at a node for the directory + * creation. When the directory is created, we remove the hashtable + * entry. + * + * Pathnames for directories also can point at the latest node that + * has been added for a file that will be created in that directory. + + */ +#include + +struct fwg; +struct fwgnode; +struct fwobject; + +struct fwg *mk_fwg(size_t budget); +// Effect: Make and return a fwg with a given budget. + +struct fwg *fwg_destroy(struct fwg *fwg); +// Effect: Waits until everything has been removed and frees the +// resources used by fwg. You may not add anything else after this +// has been invoked. + +void fwg_add1(struct fwg *fwg, + const char *name, struct fwobject *fwobject, size_t budget, + const char *prev); +// Usage: To create a file, name is the name of the file and prev is +// the name of the directory containing the file. +void fwg_add2(struct fwg *fwg, + const char *name, struct fwobject *fwobject, size_t budget, + const char *prev1, const char *prev2); +void fwg_add3(struct fwg *fwg, + const char *name, struct fwobject *fwobject, size_t budget, + const char *prev1, const char *prev2, const char *prev3); +void fwg_end_of_nodes(struct fwg *fwg); +int fwg_get_ready_node(struct fwg *fwg, + struct fwobject **fwobject, + struct fwgnode **handle); +void fwg_finish_node(struct fwg *fwg, struct fwgnode *handle); + +#endif diff --git a/software/fss-parallel-tools/gitrev.c b/software/fss-parallel-tools/gitrev.c new file mode 100644 index 0000000..d2e79a2 --- /dev/null +++ b/software/fss-parallel-tools/gitrev.c @@ -0,0 +1,2 @@ +#include "version.h" +const char gitrev[] = "++774c9c731bce9689757f2da86364f689554eb099"; diff --git a/software/fss-parallel-tools/hashfun.h b/software/fss-parallel-tools/hashfun.h new file mode 100644 index 0000000..b9a9214 --- /dev/null +++ b/software/fss-parallel-tools/hashfun.h @@ -0,0 +1,22 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef HASHFUN_H +#define HASHFUN_H +#include + +static size_t hash_init(void) { return 0; } +static size_t hash_add(size_t oldhash, size_t c) { return oldhash*11 + c; } +static size_t hash_end(size_t hash) { return (hash>>13) + hash; } +static size_t hash_string(const char *name) { + size_t r = hash_init(); + while (*name) { + r = hash_add(r, (unsigned char)*name++); + } + return hash_end(r); +} +#endif diff --git a/software/fss-parallel-tools/inodemap.c b/software/fss-parallel-tools/inodemap.c new file mode 100644 index 0000000..aed42e7 --- /dev/null +++ b/software/fss-parallel-tools/inodemap.c @@ -0,0 +1,155 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +/* Maintain a set of inodes */ +#include "inodemap.h" +#include "malloc.h" +#include +#include +#include +#include +#include + +struct entry { + bool valid; + dev_t dev; + ino_t inode; + void *value; +}; + +struct inodemap { + size_t n_entries; + size_t hash_size; + struct entry *entries; +}; + +struct inodemap *mk_inodemap(void) { + size_t initial_hash_size = 2; + struct inodemap *MALLOC(result); + struct entry *MALLOC_N(entries, initial_hash_size); + *result = (struct inodemap){.n_entries = 0, + .hash_size = initial_hash_size, + .entries = entries}; + for (size_t i = 0; i < initial_hash_size; i++) { + entries[i].valid = false; + } + return result; +} + +struct inodemap *inodemap_destroy(struct inodemap *is, + void (*destroy_value_fun)(void*)) { + assert(is); + for (size_t i = 0; i < is->hash_size; i++) { + if (is->entries[i].valid) { + destroy_value_fun(is->entries[i].value); + } + } + free(is->entries); + free(is); + return NULL; +} + +static const uint64_t FNV_OFFSET = 14695981039346656037ull; +static const uint64_t FNV_PRIME = 1099511628211ull; + +static uint64_t hash128(uint64_t a, uint64_t b) { + uint64_t hash = FNV_OFFSET; + for (size_t i = 0; i < 8; i++) { + hash ^= (a>>(i*8))&0xFFul; + hash *= FNV_PRIME; + hash ^= (b>>(i*8))&0xFFul; + hash *= FNV_PRIME; + } + return hash; +} + +uint64_t inodemap_hash(dev_t dev, ino_t inode) { + return hash128(dev, inode); +} + +static struct entry * +find_entry(struct entry *entries, size_t hash_size, dev_t dev, ino_t ino) { + // hash_size must be a power of two + assert(0 == (hash_size & (hash_size - 1))); + uint64_t h = inodemap_hash(dev, ino); + for (size_t i = 0; i < hash_size; i++) { + size_t idx = ((size_t)h + i) & (hash_size-1); + if (!entries[idx].valid + || (entries[idx].dev == dev && entries[idx].inode == ino)) { + return entries+idx; + } + } + assert(0); +} + +bool +inodemap_lookup(struct inodemap *im, + dev_t dev, + ino_t inode, + void const **value) { + struct entry *e = find_entry(im->entries, im->hash_size, dev, inode); + if (e->valid) { + *value = e->value; + } + return e->valid; + +} + +static void +maybe_grow_table(struct inodemap *im) { + if (im->n_entries * 2 < im->hash_size) { + return; + } + size_t new_size = 2*im->hash_size; + struct entry *MALLOC_N(new_entries, new_size); + for (size_t i = 0; i < new_size; i++) { + new_entries[i].valid = false; + } + for (size_t i = 0; i < im->hash_size; i++) { + if (im->entries[i].valid) { + struct entry *e = find_entry(new_entries, new_size, + im->entries[i].dev, + im->entries[i].inode); + assert(!e->valid); + *e = im->entries[i]; + } + } + free(im->entries); + im->entries = new_entries; + im->hash_size = new_size; +} + +static void +inodemap_print(struct inodemap *im, FILE *f) + __attribute__((unused)); +static void +inodemap_print(struct inodemap *im, FILE *f) { + fprintf(f, "Inode map %p. n_entries=%zd hash_size=%zd\n", + im, im->n_entries, im->hash_size); + for (size_t i = 0 ; i < im->hash_size; i++) { + if (im->entries[i].valid) { + fprintf(f, "%zd: %lu,%lu -> %p\n", i, + im->entries[i].dev, im->entries[i].inode, + im->entries[i].value); + } else { + fprintf(f, "%zd: empty\n", i); + } + } +} + +void +inodemap_insert(struct inodemap *im, dev_t dev, ino_t inode, void *value) { + maybe_grow_table(im); + // Now we know we have space. + struct entry *e = find_entry(im->entries, im->hash_size, dev, inode); + assert(!e->valid); + *e = (struct entry){.valid = true, + .dev = dev, + .inode = inode, + .value = value}; + im->n_entries++; +} diff --git a/software/fss-parallel-tools/inodemap.h b/software/fss-parallel-tools/inodemap.h new file mode 100644 index 0000000..1967ed7 --- /dev/null +++ b/software/fss-parallel-tools/inodemap.h @@ -0,0 +1,45 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef INODEMAP_H +#define INODEMAP_H +/* Maintain a set of inodes */ + +#include +#include +#include + +struct inodemap; + +// An inodemap is a mapping from pairs (dev_t, ino_t) to values (which +// are void* pointers.) + +struct inodemap *mk_inodemap(void); + +struct inodemap *inodemap_destroy(struct inodemap *im, + void (*destroy_value_fun)(void*)); +// Destroy all the + +bool +inodemap_lookup(struct inodemap *im, + dev_t dev, + ino_t inode, + void const **value); +// Effect: If there's a value in the map for (dev,inode) then return +// true and set *value to that value. Else return false. + +void +inodemap_insert(struct inodemap *im, dev_t dev, ino_t inode, void *value); +// Effect: Create a map from (dev,inode) to value. +// +// Requires: There was no map from (dev, inode) before. + + +uint64_t inodemap_hash(dev_t dev, ino_t inode); +// Compute and return a hash of dev and inode. + +#endif diff --git a/software/fss-parallel-tools/malloc.h b/software/fss-parallel-tools/malloc.h new file mode 100644 index 0000000..3a06b44 --- /dev/null +++ b/software/fss-parallel-tools/malloc.h @@ -0,0 +1,19 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef MALLOC_H +#define MALLOC_H +#include +// Suitable for doing: +// struct foo *MALLOC(v); // make something that foo can point at. +#define MALLOC(v) v = malloc(sizeof (*v)) +// Suitable for doing: +// struct foo *MALLOC_N(v, 5); // make an array of 5 "struct foo"s. +#define MALLOC_N(v, n) v = malloc(sizeof(*v)*(n)) +#define REALLOC(v, n) v = realloc(v, (n)*sizeof(*v)) +#define FREE(v) (free(v), (v) = NULL) +#endif diff --git a/software/fss-parallel-tools/parcp.c b/software/fss-parallel-tools/parcp.c new file mode 100644 index 0000000..a5586ba --- /dev/null +++ b/software/fss-parallel-tools/parcp.c @@ -0,0 +1,1215 @@ +/* + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +/* parallel copy of a directory heirarchy */ +#define _GNU_SOURCE +#include "inodemap.h" +#include "todo.h" +#include "malloc.h" +#include "putils.h" +#include "vector-of-strings.h" +#include "version.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Don't accidentally use pathcat(). Use pathcatn(). +char *pathcat(const char *a, const char *b) __attribute__((deprecated)); + +static char * +strdupn(const char *a) { + if (a == NULL) return NULL; + else return strdup(a); +} + +static char * +pathcatn(const char *a, const char *b) { + if (b == NULL) return strdupn(a); + if (a == NULL) return strdupn(b); + char *result = NULL; + int r = asprintf(&result, "%s/%s", a, b); + assert(r > 0); + return result; +} + +enum { default_n_threads = 32 }; + +// Stuff filled in by parse_args() +static struct parsed_args { + const char *cmd; + size_t n_threads; + // All of the following are set by --restore + int ignore_times; + int one_file_system; + int hard_links; + int preserve_symlinks; + int preserve_perms; + int preserve_times; + int preserve_owner; + int preserve_group; + int preserve_devices; + int preserve_specials; + int delete; + + struct vector_of_strings *vs; // the arguments + + struct vector_of_strings *exclude_from; + struct vector_of_strings *exclude; + struct vector_of_strings *include_from; + struct vector_of_strings *include; +} pargs = {.n_threads = default_n_threads}; + +/************************************************************************/ +/* Discussion of how it all works. + * + * Note: we don't have a way to delete extraneous files on the destination. + * + * We walk the source directory tree in parallel as follows: We have a global + * todo list, and we put a todo item that identifies the source and dest into + * the todo list. + * + * A worker grabs an item from the todo list. If the item is a directory, + * then use readdir to get all the directory entries. For each directory + * entry we create the destination object, and then if there's any additional + * work to do (to copy the contents), then we add that work to to the todo + * list. + * + * To copy an object from src to destination while obeying hardlinks: + * + * Copying srcname to dstname + * + * Let srcinum be the inode number of the object named by srcname. Look in + * src_inodes to see if it's already been created as filename idstname with + * inode number idstnum. + * + * If it's already been created, then we need to make a hard link from + * dstname to idstname. + + * Case 1: There is no object named dstname. Then make the link. + * + * Case 2: There is an object named dstname, and it has the same inumber + * as idstname, then we are done the link is right. + * + * Case 3: There is an object named dstname but it has a different + * inumber, so we must delete the old object (if it's a directory it must + * be an empty directory) and then make the lihnk. + * + */ + +typedef const char *magic_t; + +/************************************************************************/ +/* Keep track of the topology of inodes. */ +/************************************************************************/ +magic_t src_inode_value_magic; +struct src_inode_value { + magic_t *magic; + char *fname; + dev_t dev; + ino_t ino; +}; +static inline struct src_inode_value * +mk_src_inode_value(const char *dst_fname, dev_t dev, ino_t ino) { + struct src_inode_value *MALLOC(result); + *result = (struct src_inode_value){.magic = &src_inode_value_magic, + .fname = strdup(dst_fname), + .dev = dev, + .ino = ino}; + return result; +} +static struct src_inode_value * +src_inode_value_destroy(struct src_inode_value *siv) { + assert(siv->magic == &src_inode_value_magic); + FREE(siv->fname); + FREE(siv); + return NULL; +} + +enum { INODE_OWNER_ARRAY_SIZE = 32 }; + +static struct itopology { + pthread_mutex_t map_mutex; // protect the inodemaps during reads and writes. + struct inodemap *src_inodes; // A map from src inumber to the + // destination filename and the dst + // inumber for that destination file. + struct inodemap *dst_inodes; // A map from dst inode to a null value + // (we are using the map to maintain a + // set). + pthread_mutex_t inode_owner_array[INODE_OWNER_ARRAY_SIZE]; + // Lock based on a hash of the inumber + // and is held while the object is being + // created. Grab this lock first so + // that we can release the map_mutex + // while we create the object. +} itopology; + +static void +destroy_null(void * x) { + assert(x == NULL); +} +static void +destroy_src_inode_value_v(void *xv) { + assert(xv); + struct src_inode_value *x = xv; + assert(x->magic == &src_inode_value_magic); + src_inode_value_destroy(x); +} + +static void +init_itopology(void) { + pthread_mutex_init(&itopology.map_mutex, NULL); + itopology.src_inodes = mk_inodemap(); + itopology.dst_inodes = mk_inodemap(); + for (size_t i = 0; i < INODE_OWNER_ARRAY_SIZE; i++) { + pthread_mutex_init(&itopology.inode_owner_array[i], NULL); + } +} + +static void +deinit_itopology(void) { + pthread_mutex_destroy(&itopology.map_mutex); + itopology.src_inodes = inodemap_destroy(itopology.src_inodes, + destroy_src_inode_value_v); + itopology.dst_inodes = inodemap_destroy(itopology.dst_inodes, + destroy_null); + for (size_t i = 0; i < INODE_OWNER_ARRAY_SIZE; i++) { + pthread_mutex_destroy(&itopology.inode_owner_array[i]); + } +} + +static void +lock_itopology_ownership(dev_t dev, ino_t ino) { + uint64_t h = inodemap_hash(dev, ino); + pthread_mutex_t *om = + &itopology.inode_owner_array[h%INODE_OWNER_ARRAY_SIZE]; + pthread_mutex_lock(om); +} + +static void +unlock_itopology_ownership(dev_t dev, ino_t ino) { + uint64_t h = inodemap_hash(dev, ino); + pthread_mutex_t *om = + &itopology.inode_owner_array[h%INODE_OWNER_ARRAY_SIZE]; + pthread_mutex_unlock(om); +} + +static bool +itopology_lookup_src(dev_t dev, ino_t ino, + struct src_inode_value const ** sivp) { + pthread_mutex_lock(&itopology.map_mutex); + void const *sivv; + bool result = inodemap_lookup(itopology.src_inodes, dev, ino, &sivv); + pthread_mutex_unlock(&itopology.map_mutex); + if (result) { + assert(sivv); + struct src_inode_value const *siv = sivv; + assert(siv->magic == &src_inode_value_magic); + *sivp = siv; + } + return result; +} + +static bool +itopology_lookup_dst(dev_t dev, ino_t ino) { + pthread_mutex_lock(&itopology.map_mutex); + void const *np; + bool result = inodemap_lookup(itopology.dst_inodes, dev, ino, &np); + pthread_mutex_unlock(&itopology.map_mutex); + if (result) { + assert(np == NULL); + } + return result; +} +static void +itopology_set(dev_t sdev, + ino_t sino, + dev_t ddev, + ino_t dino, + const char *fname) { + pthread_mutex_lock(&itopology.map_mutex); + inodemap_insert(itopology.dst_inodes, ddev, dino, NULL); + inodemap_insert(itopology.src_inodes, sdev, sino, + mk_src_inode_value(fname, ddev, dino)); + pthread_mutex_unlock(&itopology.map_mutex); +} + +/******************************************************************/ +/* todo_item used to create work items for the scheduler. */ +/******************************************************************/ + +static magic_t todo_item_magic; + +enum todo_stage { TODO_ENSURE_EXISTENCE_AND_POPULATE, + TODO_REG_CONTENTS, + TODO_DIR_CONTENTS, + // Do rm -rf on dst + TODO_RM, // only dst defined (not dbuf or src or sbuf) +}; + +struct todo_item { + magic_t *magic; + enum todo_stage stage; + union { + struct todo_ensure { + const char *src_fprefix; + const char *dst_fprefix; + char *fsuffix; + } ensure; + struct todo_reg_or_dir { + const char *src_fprefix; + const char *dst_fprefix; + char *fsuffix; + struct stat sbuf; + struct stat dbuf; + } reg_or_dir; + struct todo_rm { + const char *dst_fprefix; + char *fsuffix; + char *notdirname; // just the last part of the fsuffix + size_t refcount; + struct todo_item *parent; + } rm; + } u; +}; + +static struct todo_item * +mk_todo_item_ensure_existence_and_populate(const char *src_fprefix, + const char *dst_fprefix, + char *fsuffix) +// Ownership of fsuffix gets passed to the new object +{ + struct todo_item *MALLOC(item); + *item = (struct todo_item){ + .magic = &todo_item_magic, + .stage = TODO_ENSURE_EXISTENCE_AND_POPULATE, + .u.ensure = {src_fprefix, dst_fprefix, fsuffix}}; + return item; +} + +static struct todo_item * +mk_todo_item_dir_contents(const char *src_fprefix, + const char *dst_fprefix, + char *fsuffix, + const struct stat *sbuf, + const struct stat *dbuf) +// Ownership of src and dst get passed to the new object. +{ + struct todo_item *MALLOC(item); + *item = (struct todo_item){ + .magic = &todo_item_magic, + .stage = TODO_DIR_CONTENTS, + .u.reg_or_dir = {src_fprefix, dst_fprefix, fsuffix, *sbuf, *dbuf}}; + return item; +} + +static struct todo_item * +mk_todo_item_reg_contents(const char *src_fprefix, + const char *dst_fprefix, + char *fsuffix, + const struct stat *sbuf, + const struct stat *dbuf) +// Ownership of src and dst get passed to the new object. +{ + struct todo_item *MALLOC(item); + *item = (struct todo_item){ + .magic = &todo_item_magic, + .stage = TODO_REG_CONTENTS, + .u.reg_or_dir = {src_fprefix, dst_fprefix, fsuffix, *sbuf, *dbuf}}; + return item; +} + +static struct todo_item * +mk_todo_item_rm(const char *dst_fprefix, + char *fsuffix, + char *notdirname, + struct todo_item *parent) { + struct todo_item *MALLOC(item); + *item = (struct todo_item){ + .magic = &todo_item_magic, + .stage = TODO_RM, + .u.rm = {dst_fprefix, fsuffix, notdirname, 1ul, parent}}; + return item; +} + +static struct todo_item * +todo_item_destroy(struct todo_item *item) { + assert(item->magic == &todo_item_magic); + switch (item->stage) { + case TODO_ENSURE_EXISTENCE_AND_POPULATE: { + struct todo_ensure *e = &item->u.ensure; + FREE(e->fsuffix); + break; + } + case TODO_REG_CONTENTS: + case TODO_DIR_CONTENTS: { + struct todo_reg_or_dir *rd = &item->u.reg_or_dir; + FREE(rd->fsuffix); + break; + } + case TODO_RM: { + struct todo_rm *rm = &item->u.rm; + FREE(rm->fsuffix); + FREE(rm->notdirname); + break; + } + default: + assert(0); + } + FREE(item); + return NULL; +} + +/******************************************************************/ + +static int +matches_an_exclude(const char *pathname, const char *filename) { + if (pathname) { + for (size_t i = 0; i < vector_of_strings_size(pargs.exclude); i++) { + int r = fnmatch(vector_of_strings_fetch(pargs.exclude, i), + pathname, + FNM_PATHNAME); + if (r == 0) return 1; + } + } + for (size_t i = 0; i < vector_of_strings_size(pargs.exclude); i++) { + int r = fnmatch(vector_of_strings_fetch(pargs.exclude, i), + filename, + FNM_PATHNAME); + if (r == 0) return 1; + } + return 0; +} + +static int +matches_an_include(const char *pathname, const char *filename) { + // In the absence of --include-from, always include. + if (vector_of_strings_size(pargs.include) == 0) { + return 1; + } + if (pathname) { + for (size_t i = 0; i < vector_of_strings_size(pargs.include); i++) { + int r = fnmatch(vector_of_strings_fetch(pargs.include, i), + pathname, + FNM_PATHNAME); + if (r == 0) return 1; + } + } + for (size_t i = 0; i < vector_of_strings_size(pargs.include); i++) { + int r = fnmatch(vector_of_strings_fetch(pargs.include, i), + filename, + FNM_PATHNAME); + if (r == 0) return 1; + } + return 0; +} +static void +do_ensure_existence_and_populate_item( + struct todo_list *tl, + const struct todo_ensure *item, + struct todo_item *surrounding_item); +static void +do_dir_contents_item(struct todo_list *tl, + const struct todo_reg_or_dir *item, + struct todo_item *surrounding_item); +static void +do_reg_contents_item(struct todo_list *tl, + const struct todo_reg_or_dir *item, + struct todo_item *surrounding_item); +static void +do_rm_item(struct todo_list *tl, + struct todo_rm *item, + struct todo_item *surrounding_item); +// This one isn't const because it messes around with refcount. + +static void +do_item(struct todo_list *tl, struct todo_item *item) { + // This function now owns item. It's responsible for freeing it. + // Pass ownership to one of the subfunctions: + switch (item->stage) { + case TODO_ENSURE_EXISTENCE_AND_POPULATE: + do_ensure_existence_and_populate_item(tl, &item->u.ensure, item); + break; + case TODO_REG_CONTENTS: + do_reg_contents_item(tl, &item->u.reg_or_dir, item); + break; + case TODO_DIR_CONTENTS: + do_dir_contents_item(tl, &item->u.reg_or_dir, item); + break; + case TODO_RM: + do_rm_item(tl, &item->u.rm, item); + break; + default: + assert(0); // unreachable + } +} + +pthread_mutex_t todo_rm_refcount_mutex = PTHREAD_MUTEX_INITIALIZER; + +static void +incr_refcount(struct todo_item *item) { + if (item == NULL) return; + assert(item->magic == &todo_item_magic); + assert(item->stage == TODO_RM); + pthread_mutex_lock(&todo_rm_refcount_mutex); + item->u.rm.refcount++; + pthread_mutex_unlock(&todo_rm_refcount_mutex); +} + +static void +decr_refcount(struct todo_item *item) { + if (item == NULL) return; + assert(item->magic == &todo_item_magic); + assert(item->stage == TODO_RM); + size_t count; + pthread_mutex_lock(&todo_rm_refcount_mutex); + assert(item->u.rm.refcount > 0); + count = --item->u.rm.refcount; + pthread_mutex_unlock(&todo_rm_refcount_mutex); + if (count == 0) { + char *dst = pathcatn(item->u.rm.dst_fprefix, item->u.rm.fsuffix); + if (!matches_an_exclude(item->u.rm.fsuffix, item->u.rm.notdirname) && + matches_an_include(item->u.rm.fsuffix, item->u.rm.notdirname)) { + // Don't worry if this rmdir fails. Sometimes it will fail + // because a file inside was excluded (e.g., with --exclude-from), + // and that's fine. + rmdir(dst); + } + if (item->u.rm.parent) { + decr_refcount(item->u.rm.parent); + } + todo_item_destroy(item); + FREE(dst); + } +} + +static void +do_rm_item(struct todo_list *tl, + struct todo_rm *item, + struct todo_item *surrounding_item) { + // Pass in the surrounding item so that we can maintain its reference + // count. Pass in item so that we we minimize the chance of type errors. + assert(surrounding_item->magic == &todo_item_magic); + assert(surrounding_item->stage == TODO_RM); + assert(&surrounding_item->u.rm == item); + struct stat dbuf; + char *dst = pathcatn(item->dst_fprefix, item->fsuffix); + if (!matches_an_exclude(item->fsuffix, item->notdirname) && + matches_an_include(item->fsuffix, item->notdirname)) { + MAYBE_ERROR(lstat, dst, &dbuf); + if (S_ISDIR(dbuf.st_mode)) { + DIR *d = opendir(dst); + assert(d); + struct dirent *entry; + struct vector_of_strings *fnames = mk_vector_of_strings(); + struct vector_of_strings *ndnames = mk_vector_of_strings(); + while ((entry = readdir(d))) { + char *dst_suffix = pathcatn(item->fsuffix, entry->d_name); + if (1 + && !matches_an_exclude(dst_suffix, entry->d_name) + && matches_an_include(dst_suffix, entry->d_name) + && strcmp(entry->d_name, ".") != 0 + && strcmp(entry->d_name, "..") != 0 + && strcmp(entry->d_name, ".snapshot") != 0) + { + vector_of_strings_push(fnames, dst_suffix); + vector_of_strings_push(ndnames, entry->d_name); + } + FREE(dst_suffix); + } + closedir(d); + while (vector_of_strings_size(fnames)) { + char *dst_suffix = vector_of_strings_pop(fnames); + char *ndname = vector_of_strings_pop(ndnames); + incr_refcount(surrounding_item); + struct todo_item *ti = mk_todo_item_rm(item->dst_fprefix, + dst_suffix, + ndname, + surrounding_item); + todo_list_push(tl, ti); + } + vector_of_strings_destroy(fnames); + vector_of_strings_destroy(ndnames); + } else { + MAYBE_ERROR(unlink, dst); + assert(surrounding_item->magic == &todo_item_magic); + assert(surrounding_item->stage == TODO_RM); + assert(surrounding_item->u.rm.refcount == 1); + } + } + FREE(dst); + decr_refcount(surrounding_item); + // Don't free the item here, because it might be still alive after + // decr_refcount(surrounding_item). +} + +static void +do_reg_contents_item(struct todo_list *tl __attribute__((unused)), + const struct todo_reg_or_dir *item, + struct todo_item *surrounding_item) +// Effect: Copy the contents of src to dst, both of which be existing regular +// files. We have the stat results for source and dst. We assume that the +// check for the case where mtime and size match was already done. +{ + assert(&todo_item_magic == surrounding_item->magic); + assert(TODO_REG_CONTENTS == surrounding_item->stage); + assert(&surrounding_item->u.reg_or_dir == item); + assert(S_ISREG(item->sbuf.st_mode)); + // We copy the contents of the file, truncate it, and set the permissions + // and mtime. (We assume that the the check for the case where mtime and + // size match was done before queuing the item. + char *srcname = pathcatn(item->src_fprefix, item->fsuffix); + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + int srcfd = open(srcname, O_RDONLY); + maybe_error(srcfd<0, "open", srcname, __FILE__, __LINE__); + MAYBE_ERROR(chmod, dstname, 0700); + int dstfd = open(dstname, O_WRONLY); + maybe_error(dstfd<0, "open", dstname, __FILE__, __LINE__); + size_t bufsize=1024*1024; + char *buf = malloc(bufsize); + size_t totalcount = 0; + while (1) { + ssize_t rcount = read(srcfd, buf, bufsize); + if (rcount == 0) break; + maybe_error(rcount<0, "read", srcname, __FILE__, __LINE__); + totalcount += (size_t)rcount; + char *wbuf = buf; + while (rcount > 0) { + ssize_t wcount = write(dstfd, wbuf, (size_t)rcount); + maybe_error(wcount<0, "write", dstname, __FILE__, __LINE__); + rcount -= wcount; + wbuf += (size_t)wcount; + } + } + if (item->dbuf.st_size > (off_t)totalcount) { + maybe_error(ftruncate(dstfd, (off_t)totalcount), + "ftruncate", dstname, __FILE__, __LINE__); + } + // Must do the chown before setting setuid or setgid. + int fchown_ret = fchown(dstfd, item->sbuf.st_uid, item->sbuf.st_gid); + maybe_error(fchown_ret, "fchown", dstname, __FILE__, __LINE__); + int fchmod_ret = fchmod(dstfd, item->sbuf.st_mode & (0777 | S_ISUID | S_ISGID)); + maybe_error(fchmod_ret, "fchmod", dstname, __FILE__, __LINE__); + struct timespec times[2] = {item->sbuf.st_atim, item->sbuf.st_mtim}; + int futimens_ret = futimens(dstfd, times); + maybe_error(futimens_ret, "futimens", dstname, __FILE__, __LINE__); + maybe_error(close(srcfd), "close", srcname, __FILE__, __LINE__); + maybe_error(close(dstfd), "close", dstname, __FILE__, __LINE__); + FREE(buf); + surrounding_item = todo_item_destroy(surrounding_item); + FREE(srcname); + FREE(dstname); +} + +static void +do_dir_contents_item(struct todo_list *tl, + const struct todo_reg_or_dir *item, + struct todo_item *surrounding_item) +// Effect: Copy the contents of src to dst, both of which be existing +// directories. +{ + assert(&todo_item_magic == surrounding_item->magic); + assert(TODO_DIR_CONTENTS == surrounding_item->stage); + assert(&surrounding_item->u.reg_or_dir == item); + size_t n_dirents = 0; + size_t dirents_limit = 2; + struct dirent *MALLOC_N(dirents, dirents_limit); + char *srcname = pathcatn(item->src_fprefix, item->fsuffix); + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + { + DIR *d = opendir(srcname); + if (d == NULL) fprintf(stderr, "Couldn't open %s\n", srcname); + assert(d); + struct dirent *entry; + while ((entry = readdir(d))) { + if (matches_an_exclude(item->fsuffix, entry->d_name) || + !matches_an_include(item->fsuffix, entry->d_name)) continue; + if (strcmp(entry->d_name, ".") == 0) continue; + if (strcmp(entry->d_name, "..") == 0) continue; + if (strcmp(entry->d_name, ".snapshot") == 0) continue; + if (n_dirents >= dirents_limit) { + dirents_limit*=2; + REALLOC(dirents, dirents_limit); + } + dirents[n_dirents++] = *entry; + } + closedir(d); + } + if (pargs.delete) { + //qsort(dirents, n_dirents, sizeof(*dirents), dirent_sorter); + DIR *d = opendir(dstname); + if (!d) { + fprintf(stderr, "%s:%d Could not opendir(\"%s\")\n", + __FILE__, __LINE__, dstname); + } + assert(d); + struct dirent *entry; + while ((entry = readdir(d))) { + char *new_suffix = pathcatn(item->fsuffix, entry->d_name); + if (1 + && !matches_an_exclude(new_suffix, entry->d_name) + && matches_an_include(new_suffix, entry->d_name) + && strcmp(entry->d_name, ".") != 0 + && strcmp(entry->d_name, "..") != 0 + && strcmp(entry->d_name, ".snapshot") != 0) + { + for (size_t i = 0; i < n_dirents; i++) { + if (strcmp(entry->d_name, dirents[i].d_name) == 0) { + FREE(new_suffix); + goto keep_it; + } + } + todo_list_push(tl, + mk_todo_item_rm( + item->dst_fprefix, + new_suffix, + strdup(entry->d_name), + NULL)); + } else { + FREE(new_suffix); + } + keep_it: ;/* nothing */ + } + closedir(d); + } + for (size_t i = 0; i < n_dirents; i++) { + char *new_suffix = pathcatn(item->fsuffix, dirents[i].d_name); + // Ensure the existence immediately. If needed, this will create an + // additional task to populate the item. + if (!matches_an_exclude(new_suffix, dirents[i].d_name) && + matches_an_include(new_suffix, dirents[i].d_name)) { + do_item(tl, + mk_todo_item_ensure_existence_and_populate( + item->src_fprefix, + item->dst_fprefix, + new_suffix)); + } else { + FREE(new_suffix); + } + } + // now set the permissions on the dir. + struct stat sbuf; + MAYBE_ERROR(lstat, srcname, &sbuf); + int chown_ret = chown(dstname, sbuf.st_uid, sbuf.st_gid); + maybe_error(chown_ret, "chown", dstname, __FILE__, __LINE__); + int chmod_ret = chmod(dstname, sbuf.st_mode & (0777 | S_ISUID | S_ISGID)); + maybe_error(chmod_ret, "chmod", dstname, __FILE__, __LINE__); + struct timespec times[2] = {sbuf.st_atim, sbuf.st_mtim}; + int utimensat_ret = utimensat(AT_FDCWD, dstname, times, 0); + maybe_error(utimensat_ret, "utimensat", dstname, __FILE__, __LINE__); + FREE(dirents); + todo_item_destroy(surrounding_item); + FREE(srcname); + FREE(dstname); +} + +static void +do_ensure_dir(struct todo_list *tl, + const struct todo_ensure *item, + const struct stat *sbuf, + bool dexists, + const struct stat *dbuf); +static void +do_ensure_symlink(struct todo_list *tl, + const struct todo_ensure *item, + const struct stat *sbuf, + bool dexists, + const struct stat *dbuf); +static void +do_ensure_nod(struct todo_list *tl, + const struct todo_ensure *item, + const struct stat *sbuf, + bool dexists, + struct stat *dbuf); +void +do_create_item_obj(struct todo_list *tl, + struct todo_item *item, + bool dexists); + +static void +do_ensure_existence_and_populate_item( + struct todo_list *tl, + const struct todo_ensure *item, + struct todo_item *surrounding_item) +// Effect: Ensure that the dst is created, and then if it is a directory or +// regular file enqueue an item to copy the contents. +{ + assert(&todo_item_magic == surrounding_item->magic); + assert(TODO_ENSURE_EXISTENCE_AND_POPULATE == surrounding_item->stage); + assert(&surrounding_item->u.ensure == item); + struct stat sbuf; + char *srcname = pathcatn(item->src_fprefix, item->fsuffix); + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + int er = lstat(srcname, &sbuf); + if ( er != 0 && errno == ENOENT) { + fprintf(stderr, "%s:%d Warning: Could not lstat(\"%s\"). errno=%d (%s)\n", + __FILE__, __LINE__, srcname, errno, strerror(errno)); + surrounding_item = todo_item_destroy(surrounding_item); + FREE(srcname); + FREE(dstname); + return; + } + + bool dexists; + struct stat dbuf; + { + int r = lstat(dstname, &dbuf); + if (r != 0 && errno == ENOENT) { + dexists = false; + } else if (r == 0) { + dexists = true; + } else { + maybe_error(r, "lstat", dstname, __FILE__, __LINE__); + assert(0); // unreachable + } + } + if (S_ISDIR(sbuf.st_mode)) { + do_ensure_dir(tl, item, &sbuf, dexists, &dbuf); + } else if (S_ISLNK(sbuf.st_mode)) { + do_ensure_symlink(tl, item, &sbuf, dexists, &dbuf); + } else if (0 + || S_ISREG(sbuf.st_mode) + || S_ISCHR(sbuf.st_mode) + || S_ISBLK(sbuf.st_mode) + || S_ISFIFO(sbuf.st_mode) + || S_ISSOCK(sbuf.st_mode)) { + do_ensure_nod(tl, item, &sbuf, dexists, &dbuf); + } else { + assert(0); // unreachable + } + surrounding_item = todo_item_destroy(surrounding_item); + FREE(srcname); + FREE(dstname); +} + +static void +do_ensure_dir(struct todo_list *tl, + const struct todo_ensure *item, + const struct stat *sbuf, + bool dexists, + const struct stat *dbuf) { + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + if (!dexists) { + // Dest doesn't exist. Create it and copy contents (which will + // also fix the permissions of the directory). + mkdir(dstname, 0770); + } else if (S_ISDIR(dbuf->st_mode)) { + // Dest exists as a dir. Make it writeable, copy contents (and + // fix permissions). + MAYBE_ERROR(chmod, dstname, 0770); + } else { + // Dest exists, but it's not a directory. Get rid of it, make the + // dir, and copy contents. + MAYBE_ERROR(unlink, dstname); + mkdir(dstname, 0770); + } + // We could try to reuse the item, but that's not very clean. + todo_list_push(tl, mk_todo_item_dir_contents(item->src_fprefix, + item->dst_fprefix, + strdupn(item->fsuffix), + sbuf, + dbuf)); + FREE(dstname); +} + +static void +unlink_and_symlink(const struct todo_ensure *item, const char *target) { + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + MAYBE_ERROR(unlink, dstname); + MAYBE_ERROR2(symlink, target, dstname); + FREE(dstname); +} + +static void +do_ensure_symlink(struct todo_list *tl __attribute__((unused)), + const struct todo_ensure *item, + const struct stat *sbuf, + bool dexists, + const struct stat *dbuf) { + char *srcname = pathcatn(item->src_fprefix, item->fsuffix); + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + size_t ssize = (size_t)sbuf->st_size; + char *slinkval = malloc(ssize + 1); + ssize_t sgot = readlink(srcname, slinkval, ssize+1); + maybe_error(sgot < 0 || (size_t)sgot != ssize, + "readlink", srcname, + __FILE__, __LINE__); + slinkval[ssize]=0; + if (!dexists) { + MAYBE_ERROR2(symlink, slinkval, dstname); + } else if (S_ISDIR(dbuf->st_mode)) { + delete_dir_recursively(dstname); + MAYBE_ERROR2(symlink, slinkval, dstname); + } else if (S_ISLNK(dbuf->st_mode)) { + if (dbuf->st_size != sbuf->st_size) { + unlink_and_symlink(item, slinkval); + } else { + size_t size = (size_t)dbuf->st_size; + char *dlinkval = malloc(size + 1); + + ssize_t dgot = readlink(dstname, dlinkval, size+1); + maybe_error(dgot < 0 || (size_t)dgot != size, + "readlink", dstname, + __FILE__, __LINE__); + int cmp = memcmp(dlinkval, slinkval, size); + FREE(dlinkval); + if (cmp != 0) { + unlink_and_symlink(item, slinkval); + } else { + // The existing symlink is what we want. + /*nothing*/; + } + } + } else { + unlink_and_symlink(item, slinkval); + } + FREE(srcname); + FREE(dstname); + FREE(slinkval); +} + +static void +do_ensure_nod(struct todo_list *tl, + const struct todo_ensure *item, + const struct stat *sbuf, + bool dexists, + struct stat *dbuf) +// Effect: Make sure that a (nondir) file object exists. If it's a regular +// file then add a todo item to populate it. +{ + char *srcname = pathcatn(item->src_fprefix, item->fsuffix); + char *dstname = pathcatn(item->dst_fprefix, item->fsuffix); + // Here we have to do some extra work to make the topology of the inodes + // match in the source and destionation. The first question is whether + // the source inode has a corresponding destination inode already. + lock_itopology_ownership(sbuf->st_dev, sbuf->st_ino); + struct src_inode_value const * siv; + bool already_copied = itopology_lookup_src(sbuf->st_dev, + sbuf->st_ino, + &siv); + // The om is still held, so no one else can copy the inode until we + // resolve this. + + if (already_copied) { + // The file has already been copied, so all we need is a link. It's + // resolved so we can release om. + unlock_itopology_ownership(sbuf->st_dev, sbuf->st_ino); + if (dexists) { + // The destination already exists. + if (S_ISDIR(dbuf->st_mode)) { + // The destination exists, but it's a dir. Must rmdir the + // directory, then make a link to the object. + MAYBE_ERROR(rmdir, dstname); + goto dest_doesnt_exist; + } else if (1 + && siv->dev == dbuf->st_dev + && siv->ino == dbuf->st_ino) { + // The link exists to the right thing, so we are done. + /*nothing*/ ; + } else { + // Something exists there, but it's the wrong thing. + MAYBE_ERROR(unlink, dstname); + goto dest_doesnt_exist; + } + } else { + // The destination doesn't exist. + dest_doesnt_exist: + MAYBE_ERROR2(link, siv->fname, dstname); + } + } else { + // The file has not been copied, so we need to make sure the + // destination exists and isn't an inode that *has* been previously + // mapped. + if (dexists) { + // The destination exists. + if (S_ISDIR(dbuf->st_mode)) { + // The destination exists, but it's a dir. Must rmdir the + // directory, then make a link to the object. + MAYBE_ERROR(rmdir, dstname); + goto destination_doesnt_exist; + } else if (0 + || ((dbuf->st_mode & S_IFMT) + != (sbuf->st_mode & S_IFMT)) + || itopology_lookup_dst(dbuf->st_dev, dbuf->st_ino)) { + // It's a different file type or the dev and inode already has + // been mapped, so we cannot use the current destination. + MAYBE_ERROR(unlink, dstname); + goto destination_doesnt_exist; + } else { + // dest exists, and we can reuse it. + // Todo: If we don't have write permission, make it writeable. + /*nothing*/ ; + } + } else { + // The destination doesn't exist. + destination_doesnt_exist: + MAYBE_ERROR(mknod, + dstname, + sbuf->st_mode | 0700, + sbuf->st_dev); + MAYBE_ERROR(lstat, dstname, dbuf); + } + // Now we have the right destination inode. + itopology_set(sbuf->st_dev, sbuf->st_ino, + dbuf->st_dev, dbuf->st_ino, + dstname); + if (S_ISREG(sbuf->st_mode) + && (0 + || sbuf->st_size != dbuf->st_size + || sbuf->st_mtim.tv_sec != dbuf->st_mtim.tv_sec + || sbuf->st_mtim.tv_nsec != dbuf->st_mtim.tv_nsec)) + { + todo_list_push(tl, mk_todo_item_reg_contents(item->src_fprefix, + item->dst_fprefix, + strdupn(item->fsuffix), + sbuf, + dbuf)); + } + unlock_itopology_ownership(sbuf->st_dev, sbuf->st_ino); + } + FREE(srcname); + FREE(dstname); +} + +static void do_cp(const char *src, const char *dst) { + struct todo_item *item = mk_todo_item_ensure_existence_and_populate( + src, dst, NULL); + struct todo_list *tl = mk_todo_list(); + todo_list_run(tl, pargs.n_threads, item, do_item); + todo_list_destroy(tl); +} + +void help(int exitcode) { + FILE *out = exitcode ? stderr : stdout; + fprintf(out, "Usage: %s [OPTION] source dest\n", pargs.cmd); + fprintf(out, "Recursive copy of a directory hierarchy, done in parallel for performance.\n"); + fprintf(out, " OPTION can be\n"); + fprintf(out, " -P P (capital P) use P-fold parallelism (default %d threads).\n", default_n_threads); + fprintf(out, " --restore Recurse, put stop at file system boundaries, preserve hard links, symlinks permissions, modification times, group, owners, and special files such as named sockets and fifos. Similar to `-arxH` in rsync. Note: This does not include --delete.\n"); + fprintf(out, " --delete Delete files that appear in dst but not src.\n"); + fprintf(out, " --exclude-from=FILE FILE contains a list of glob(3)-style patterns, which will not be copied.\n"); + fprintf(out, " --include-from=FILE FILE contains a list of glob(3)-style patterns, which will be copied. Everything else will be omitted.\n"); + fprintf(out, " -- remaining arguments are files even if they start with '-'.\n"); + fprintf(out, " -h, --help print help.\n"); + fprintf(out, " --version print program version\n"); + fprintf(out, " --history print a brief history of the versions\n"); + fprintf(out, "This is version %s gitrev %s compiled %s %s.\n", + VERSION, gitrev, __DATE__, __TIME__); + fprintf(out, " Report bugs to parallel-tools-support_ww_grp@oracle.com\n"); + exit(exitcode); +} + +static int prefix_matches(const char *prefix, const char *string, char const **suffix) { + while (*prefix == *string) { + prefix++; string++; + } + if (*prefix == 0) { + *suffix = string; + return 1; + } else { + return 0; + } +} + +static void +parse_args(int argc, const char *argv[], struct vector_of_strings *args) { + assert(argc>0); + pargs.cmd = argv[0]; + argc--; argv++; + while (argc) { + char const *suffix; + if (strcmp(argv[0], "-P") == 0) { + argc--; argv++; + if (argc == 0) help(1); + pargs.n_threads = parse_number_or_help(argv[0]); + } else if (strcmp(argv[0], "--delete") == 0) { + pargs.delete = 1; + } else if (strcmp(argv[0], "--restore") == 0) { + pargs.ignore_times = 1; + pargs.one_file_system = 1; + pargs.hard_links = 1; + pargs.preserve_symlinks = 1; + pargs.preserve_perms = 1; + pargs.preserve_times = 1; + pargs.preserve_owner = 1; + pargs.preserve_group = 1; + pargs.preserve_devices = 1; + pargs.preserve_specials = 1; + } else if (prefix_matches("--exclude-from=", argv[0], &suffix)) { + if (vector_of_strings_size(pargs.include_from)) { + fprintf(stderr, "%s\n", + "Options --exclude-from and --include-from are mutually exclusive."); + help(1); + } + vector_of_strings_push(pargs.exclude_from, suffix); + } else if (prefix_matches("--include-from=", argv[0], &suffix)) { + if (vector_of_strings_size(pargs.exclude_from)) { + fprintf(stderr, "%s\n", + "Options --exclude-from and --include-from are mutually exclusive."); + help(1); + } + vector_of_strings_push(pargs.include_from, suffix); + } else if (strcmp(argv[0], "-h") == 0 || strcmp(argv[0], "--help") == 0) { + help(0); + } else if (strcmp(argv[0], "--version") == 0) { + printf("%s %s\n" + "Copyright (C) 2019 Oracle.\n" + "Written by Bradley C. Kuszmaul\n", + pargs.cmd, VERSION); + exit(0); + } else if (strcmp(argv[0], "--history") == 0) { + fprintf(stderr, "%s", VERSION_HISTORY); + exit(0); + } else if (strcmp(argv[0], "--") == 0) { + while (1) { + argc--; argv++; + vector_of_strings_push(args, argv[0]); + if (argc==1) break; + } + } else { + vector_of_strings_push(args, argv[0]); + } + argc--; argv++; + } + if (vector_of_strings_size(args) != 2) { + fprintf(stderr, "Need two arguments for copying, got %zu\n", + vector_of_strings_size(args)); + help(1); + } + { + char *line = NULL; + size_t len = 0; + ssize_t nread = 0; + for (size_t i = 0; i < vector_of_strings_size(pargs.exclude_from); i++) { + char const *fname = vector_of_strings_fetch(pargs.exclude_from, i); + FILE *f = fopen(fname, "r"); + if (f == NULL) { + fprintf(stderr, "Could not open file named %s, " + "mentioned as an --exclude-from\n", + fname); + FREE(line); + exit(1); + } + while ((nread = getline(&line, &len, f)) != -1) { + size_t llen = strlen(line); + if (llen == 0) continue; + if (';' == line[0]) continue; + if ('#' == line[0]) continue; + if (line[llen-1] == '\n') line[llen-1]=0; + vector_of_strings_push(pargs.exclude, line); + } + fclose(f); + } + for (size_t i = 0; i < vector_of_strings_size(pargs.include_from); i++) { + char const *fname = vector_of_strings_fetch(pargs.include_from, i); + FILE *f = fopen(fname, "r"); + if (f == NULL) { + fprintf(stderr, "Could not open file named %s, " + "mentioned as an --include-from\n", + fname); + FREE(line); + exit(1); + } + while ((nread = getline(&line, &len, f)) != -1) { + size_t llen = strlen(line); + if (llen == 0) continue; + if (';' == line[0]) continue; + if ('#' == line[0]) continue; + if (line[llen-1] == '\n') line[llen-1]=0; + vector_of_strings_push(pargs.include, line); + } + fclose(f); + } + FREE(line); + } +} + +static void on_exit_destroy_parcp(int status __attribute__((unused)), + void *ignore __attribute__((unused))) { + vector_of_strings_destroy(pargs.exclude_from); + vector_of_strings_destroy(pargs.exclude); + vector_of_strings_destroy(pargs.include_from); + vector_of_strings_destroy(pargs.include); + vector_of_strings_destroy(pargs.vs); + deinit_itopology(); +} + +int main (int argc, const char *argv[]) { + pargs.vs = mk_vector_of_strings(); + pargs.exclude_from = mk_vector_of_strings(); + pargs.exclude = mk_vector_of_strings(); + pargs.include_from = mk_vector_of_strings(); + pargs.include = mk_vector_of_strings(); + init_itopology(); + on_exit(on_exit_destroy_parcp, NULL); + parse_args(argc, argv, pargs.vs); + assert(vector_of_strings_size(pargs.vs) == 2); + char *dst = vector_of_strings_pop(pargs.vs); + char *src = vector_of_strings_pop(pargs.vs); + struct stat statb; + { + int r = lstat(src, &statb); + if (r != 0) { + fprintf(stderr, "Could not lstat(\"%s\") src. errno=%d (%s)\n", + src, errno, strerror(errno)); + FREE(dst); + FREE(src); + exit(1); + } + if (!S_ISDIR(statb.st_mode)) { + fprintf(stderr, "%s is not a directory.\n", src); + fprintf(stderr, "parcp operates only on directories. To copy a single file, use cp\n"); + FREE(dst); + FREE(src); + exit(1); + } + } + { + int r = mkdir(dst, S_IRUSR|S_IWUSR|S_IXUSR); + if (r != 0 && errno != EEXIST) { + fprintf(stderr, "Could not mkdir(\"%s\") dst. errno=%d (%s)\n", + dst, errno, strerror(errno)); + FREE(dst); + FREE(src); + exit(1); + } + } + // Creating a copy of src to pass to basename function + // as it may modify the input. + char *src_copy = strdup(src); + + // Using POSIX version of basename routine(picked from '') + // instead of GNU verison(picked from ''). + + // Including '' either before or after including '', + // will override the GNU variant of basename(). + char *src_basename = basename(src_copy); + if (src_basename == NULL) { + fprintf(stderr,"Could not extract basename of src \"%s\"", src_basename); + FREE(dst); + FREE(src); + exit(1); + } + char *old_dest = dst; + dst = pathcatn(old_dest, src_basename); + free(old_dest); + + do_cp(src, dst); + free(dst); + free(src); + free(src_copy); + return 0; +} diff --git a/software/fss-parallel-tools/parrm.c b/software/fss-parallel-tools/parrm.c new file mode 100644 index 0000000..dd22e84 --- /dev/null +++ b/software/fss-parallel-tools/parrm.c @@ -0,0 +1,210 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "malloc.h" +#include "putils.h" +#include "version.h" +#include "sched.h" +#include "vector-of-strings.h" +#include +#include +#include +#include +#include +#include +#include + +static const char *cmd; +enum { default_n_threads = 32 }; +static size_t n_threads = default_n_threads; + +void help(int exitcode) { + FILE *out = exitcode ? stderr : stdout; + fprintf(out, "Usage: %s [OPTION]... [DIR]...\n", cmd); + fprintf(out, "Recursive remove of a directory hierarchy, done in parallel for performance.\n"); + fprintf(out, " OPTION can be\n"); + fprintf(out, " -h, --help print help.\n"); + fprintf(out, " -P P (capital P) use P-fold parallelism (default %d threads).\n", default_n_threads); + fprintf(out, " -- remaining arguments are files even if they start with '-'.\n"); + fprintf(out, " --version print program version\n"); + fprintf(out, " --history print a brief history of the versions\n"); + fprintf(out, "This is version %s gitrev %s compiled %s %s.\n", + VERSION, gitrev, __DATE__, __TIME__); + fprintf(out, "Report bugs to parallel-tools-support_ww_grp@oracle.com\n"); + exit(exitcode); +} + +static magic_t rm_magic = "rm magic"; + +struct rmframe { + magic_t *magic; + char *filename; + struct vector_of_strings *files_in_dir; + size_t inlet_sum; + size_t sum; +}; + +static struct rmframe * +mk_rmframe(char *filename) { + struct rmframe *MALLOC(rf); + *rf = (struct rmframe){.magic = &rm_magic, + .filename = filename, + .files_in_dir = NULL, + .inlet_sum = 0, + .sum = 0}; + return rf; +} + +static struct rmframe *rmframe_free(struct rmframe *rf) { + FREE(rf->filename); + if (rf->files_in_dir) { + rf->files_in_dir = vector_of_strings_destroy(rf->files_in_dir); + } + FREE(rf); + return NULL; +} + +static struct rmframe *rmframe_cast(void *app_frame) { + struct rmframe *rf = app_frame; + assert(rf->magic == &rm_magic); + return rf; +} + +static void rmfun_return_inlet(void *parent_app_frame, size_t result) { + struct rmframe *rf = rmframe_cast(parent_app_frame); + rf->inlet_sum += result; +} + + +static struct frame *rmfun_have_dirlist(struct frame *frame, void *app_frame); +static struct frame *rmfun_sync(struct frame *frame, void *app_frame); +static struct frame *rmfun(struct frame *frame, void *app_frame) { + struct rmframe *rf = rmframe_cast(app_frame); + struct stat statbuf; + { + int r = lstat(rf->filename, &statbuf); + if (r != 0) { + fprintf(stderr, "%s: cannot stat '%s': %s\n", cmd, rf->filename, strerror(errno)); + rf->sum++; + return sched_sync(frame, rmfun_sync); + } + } + if (S_ISDIR(statbuf.st_mode)) { + assert(rf->files_in_dir == NULL); + rf->files_in_dir = mk_vector_of_strings(); + DIR *dir = opendir(rf->filename); + if (dir == NULL) { + fprintf(stderr, "%s: A cannot remove '%s': %s\n", cmd, rf->filename, strerror(errno)); + rf->sum++; + return sched_sync(frame, rmfun_sync); + } else { + struct dirent *entry; + while ((entry = readdir(dir))) { + if (strcmp(entry->d_name, ".") == 0) continue; + if (strcmp(entry->d_name, "..") == 0) continue; + char *path = pathcat(rf->filename, entry->d_name); + vector_of_strings_push(rf->files_in_dir, path); + FREE(path); + } + closedir(dir); + return rmfun_have_dirlist(frame, app_frame); + } + } else { + if (0) { + printf("sleep start\n"); + sleep(2); + printf("sleep end\n"); + } + int r = unlink(rf->filename); + if (r != 0) { + fprintf(stderr, "%s: B cannot remove '%s': %s\n", cmd, rf->filename, strerror(errno)); + rf->sum++; + } + return sched_sync(frame, rmfun_sync); + } +} +static struct frame *rmfun_have_dirlist(struct frame *frame, void *app_frame) { + struct rmframe *rf = rmframe_cast(app_frame); + assert(rf->files_in_dir != NULL); + char *fname; + while ((fname = vector_of_strings_pop(rf->files_in_dir))) { + struct rmframe *subframe = mk_rmframe(fname); + return rmfun(sched_spawn(frame, rmfun_have_dirlist, subframe, rmfun_return_inlet), + subframe); + } + return sched_sync(frame, rmfun_sync); +} + +static struct frame *rmfun_sync(struct frame *frame, void *app_frame) { + struct rmframe *rf = rmframe_cast(app_frame); + size_t sum = rf->inlet_sum + rf->sum; + if (rf->filename && rf->files_in_dir && sum == 0) { + // If NULL then there is no dir. We don't delete the parents + // of the files mentioned on the command line argument. + int r = rmdir(rf->filename); + if (r != 0) { + fprintf(stderr, "%s: C cannot remove '%s': %s\n", cmd, rf->filename, strerror(errno)); + sum++; + } + } + rf = rmframe_free(rf); + return sched_return(frame, sum); +} + +static int do_rm(struct vector_of_strings *fargs) { + struct rmframe *rf = mk_rmframe(NULL); + assert(rf->files_in_dir == NULL); + rf->files_in_dir = fargs; + size_t n = prun(n_threads, rmfun_have_dirlist, rf); + if (0) fprintf(stderr, "Couldn't delete %zu objects\n", n); + return n==0; +} + + +static void +parse_args(int argc, const char *argv[], struct vector_of_strings *fargs) { + assert(argc > 0); + cmd = argv[0]; + argc--; argv++; + while (argc) { + if (strcmp(argv[0], "-P") == 0) { + argc--; argv++; + if (argc == 0) help(1); + n_threads = parse_number_or_help(argv[0]); + } else if (strcmp(argv[0], "-h") == 0 || + strcmp(argv[0], "--help") == 0) { + help(0); + } else if (strcmp(argv[0], "--version") == 0) { + printf("%s %s\n" + "Copyright (C) 2019 Oracle.\n" + "Written by Bradley C. Kuszmaul\n", + cmd, VERSION); + exit(0); + } else if (strcmp(argv[0], "--history") == 0) { + fprintf(stderr, "%s", VERSION_HISTORY); + exit(0); + } else if (strcmp(argv[0], "--") == 0) { + argc--; argv++; + while (argc) { + vector_of_strings_push(fargs, argv[0]); + argc--; argv++; + } + } else { + vector_of_strings_push(fargs, argv[0]); + } + argc--; argv++; + } +} + +int main (int argc, const char *argv[]) { + struct vector_of_strings *fargs = mk_vector_of_strings(); + parse_args(argc, argv, fargs); + if (!do_rm(fargs)) { + exit(1); + } + return 0; +} diff --git a/software/fss-parallel-tools/partar.c b/software/fss-parallel-tools/partar.c new file mode 100644 index 0000000..6504ba3 --- /dev/null +++ b/software/fss-parallel-tools/partar.c @@ -0,0 +1,1893 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#define _GNU_SOURCE +#include "fdleak.h" +#include "vec.h" +#include "putils.h" +#include "fwg.h" +#include "inodemap.h" +#include "malloc.h" +#include "sched.h" +#include "stringset.h" +#include "vector-of-strings.h" +#include "version.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/spew.h" +#include + + +// Parallel tar. For parallel untar The main thread scans through the +// tar file. If it finds a directory, it creates it. If it finds a +// regular file, it mallocs a place to hold the contents of the file, +// scans everything into the alloced object, and stores the alloced +// object into a vector of items to work on. (The main thread also +// handles symlinks: rationale symlinks seem to be uncommon, and it's +// simpler this way.) +// +// Meanwhile, a bunch of threads grab items out of the vector. Each +// item involves creating a file and writing the bytes into the file. +// They grab a random item to reduce the chances that they are trying +// to create files in the same directory. +// +// We cannot handle the case where the same file appears twice in the +// tar file (presumably the semantics is that the second file "wins".) +// We use a hash table to detect that this happens. The hash table +// doesn't need a mutex since it's only accessed by the main thread. + +/* tar Header Block, from POSIX 1003.1-1990. */ + +/* POSIX header. */ + +struct posix_header +{ /* byte offset */ + char name[100]; /* 0 */ + char mode[8]; /* 100 */ + char uid[8]; /* 108 */ + char gid[8]; /* 116 */ + char size[12]; /* 124 */ + char mtime[12]; /* 136 */ + char chksum[8]; /* 148 */ + char typeflag[1]; /* 156 */ + char linkname[100]; /* 157 */ + char magic[6]; /* 257 */ + char version[2]; /* 263 */ + char uname[32]; /* 265 */ + char gname[32]; /* 297 */ + char devmajor[8]; /* 329 */ + char devminor[8]; /* 337 */ + char prefix[155]; /* 345 */ + /* 500 */ + char pad[12]; +}; + +static void print_byte(char c) { + if (isprint(c) && c != '\\') { + fprintf(stderr, "%c", c); + } else { + fprintf(stderr, "\\%03o", c); + } +} + +static void print_bytes(const char *s, size_t len) { + fprintf(stderr, "\""); + for (size_t i = 0; i < len; i++) { + print_byte(s[i]); + } + fprintf(stderr, "\""); +} + +#define PRINT_BYTES(name) (fprintf(stderr, "\n %8s = ", #name), print_bytes(h->name, sizeof(h->name))) + +static void debug_print_posix_header(const struct posix_header *h) { + fprintf(stderr, "posix header {"); + PRINT_BYTES(name); + PRINT_BYTES(mode); + PRINT_BYTES(uid); + PRINT_BYTES(gid); + PRINT_BYTES(size); + PRINT_BYTES(mtime); + PRINT_BYTES(chksum); + PRINT_BYTES(typeflag); + PRINT_BYTES(linkname); + PRINT_BYTES(magic); + PRINT_BYTES(version); + PRINT_BYTES(uname); + PRINT_BYTES(gname); + PRINT_BYTES(devmajor); + PRINT_BYTES(devminor); + PRINT_BYTES(prefix); + PRINT_BYTES(pad); + fprintf(stderr, "\n ... }\n"); +} + +enum { default_memory_budget_MiB = 100 }; +enum { MiB = 1024*1024 }; +static size_t memory_budget_MiB = default_memory_budget_MiB; + +enum { max_name_length = 255 }; + +enum { + DONE = 0, + ERROR = 1, + CONTINUE = 2, +}; + +#define ROOT_UID 0 +static int we_are_root; + +static int arg_preserve = 0; // default is 1 if we are root +static int arg_gunzip = 0; +static int arg_show_progress = 0; +static const char *arg_named_input = NULL; +static const char *arg_change_dir = NULL; + +pthread_cond_t fwcond = PTHREAD_COND_INITIALIZER; +pthread_mutex_t fwmutex = PTHREAD_MUTEX_INITIALIZER; + +struct fwobject { + char command; + char *fullname; + char *linkname; + size_t size; + size_t mode; + size_t uid; + size_t gid; + size_t mtime; + // If bytes!=NULL then the data is in bytes. + char *bytes; + // + // Else it's in input_file, to be gotten by do_read(input_file, + // size size_to_read), set *done and signal fwcond when done. + FILE *input_file; + size_t size_to_read; + int *done; +}; +static struct fwobject *mk_fwobject(char command, + const char *fullname, + const char *linkname, + size_t size, + size_t mode, + size_t uid, + size_t gid, + size_t mtime, + char *bytes/*[size]*/) { + struct fwobject *MALLOC(b); + *b = (struct fwobject){.command = command, + .fullname = fullname ? strdup(fullname) : NULL, + .linkname = linkname ? strdup(linkname) : NULL, + .size = size, + .mode = mode, + .uid = uid, + .gid = gid, + .mtime = mtime, + .bytes = bytes, + .input_file = NULL, + .size_to_read = 0, + .done = NULL}; + return b; +} + +static struct fwobject *mk_fwobject_read_from_input(char command, + const char *fullname, + size_t size, + size_t mode, + size_t uid, + size_t gid, + size_t mtime, + FILE *input_file, + size_t size_to_read, + int *done) { + struct fwobject *MALLOC(b); + *b = (struct fwobject){.command = command, + .fullname = fullname ? strdup(fullname) : NULL, + .linkname = NULL, + .size = size, + .mode = mode, + .uid = uid, + .gid = gid, + .mtime = mtime, + .bytes = NULL, + .input_file = input_file, + .size_to_read = size_to_read, + .done = done}; + return b; +} + + +static magic_t mep_magic = "mep magic"; + +struct mep_extra { + magic_t *magic; + const char *fname; +}; + +static int matches_excludes_predicate(const char *pattern, + void *mepv) { + struct mep_extra *mep = mepv; + assert(mep->magic == &mep_magic); + spew(SPEW_DEBUG, "matching %s to %s", mep->fname, pattern); + + const char *str = mep->fname; + while (*str) { + spew(SPEW_DEBUG, "str=%s", str); + const char *end = index(str, '/'); + if (end == NULL) { + spew(SPEW_DEBUG, "matching %s to %s", str, pattern); + int r = fnmatch(pattern, str, 0); + spew(SPEW_DEBUG, "%d", !r); + return !r; + } else if (strcmp(end, str) == 0) { + str = end+1; + spew(SPEW_DEBUG, "str=%s end=%s", str,end); + } else { + char path[256]; + assert(end > str); + assert((size_t)(end-str) < sizeof(path) - 1); + memcpy(path, str, (size_t)(end-str)); + path[end-str] = 0; + spew(SPEW_DEBUG, "matching %s to %s", path, pattern); + if (fnmatch(pattern, path, 0) == 0) { + spew(SPEW_DEBUG, "%s matches %s (original %s)", path, pattern, mep->fname); + return 1; + } + str = end+1; + spew(SPEW_DEBUG, "now str=%s", str); + } + } + spew(SPEW_DEBUG, "nope"); + return 0; +} + +int some_errors = 0; + +static struct vector_of_strings *excludes; +static int matches_excludes(const char *fname) { + struct mep_extra mep = {&mep_magic, fname}; + return vector_of_strings_any(excludes, + matches_excludes_predicate, + &mep); +} + +static struct fwg *fwg; + +static void pinit(void) { + fwg = mk_fwg(memory_budget_MiB * 1024ul * 1024ul); +}; +static void pdestroy(void) { + fwg = fwg_destroy(fwg); +} + +static size_t smin(size_t a, size_t b) { + if (a < b) return a; + else return b; +} + +/* + * The function checks the stat of the files that already exist. + * It then compares the type of the existing file with the new file + * that needs to be created at the same path. In case of mismatch + * the existing file is deleted. If the existing file type is + * directory, it deletes the subtree as well. + */ +static int check_and_delete_conflicts(char typeflag, char *fullname) { + struct stat filestat; + char tempname[600]; + snprintf(tempname, sizeof(tempname), "%s", + fullname); + if (tempname[strlen(tempname) - 1] == '/') { + tempname[strlen(tempname) - 1] = 0; + } + int ret = stat(tempname, &filestat); + if (ret == 0) { + if (S_ISDIR(filestat.st_mode) && typeflag != '5') { + /* + * Found a directory at the path where a new type + * of file is expected. Deleting along with subtree. + */ + delete_dir_recursively(tempname); + } else if ((S_ISREG(filestat.st_mode) && typeflag != '0') + || (S_ISLNK(filestat.st_mode) && typeflag != '2') + || (S_ISFIFO(filestat.st_mode) && typeflag != '6')) + { + do_unlink(tempname); + } + } + return 0; +} + +static void process_fwobject(size_t threadn, struct fwobject *b, struct fwgnode *handle, int retry) { + int ret; + spew(SPEW_DEBUG, "cmd='%c' %s handle=%p", b->command, b->fullname, handle); + switch (b->command) { + case '0': { + // create file + spew(SPEW_DEBUG, "%lu: parallel (small) regular file \"%s\"", threadn, b->fullname); + chmod(b->fullname, 0700); + FILE *of = do_creat(b->fullname); + if (of == NULL && retry == 1) { + check_and_delete_conflicts(b->command, b->fullname); + process_fwobject(threadn, b, handle, 0); + return; + } + if (b->bytes) { + assert(!b->input_file && !b->size_to_read && !b->done); + do_write(of, b->bytes, b->size); + } else { + spew(SPEW_DEBUG, "Immediate read of %lu bytes into %lu bytes", b->size_to_read, b->size); + assert(b->input_file && b->size_to_read && b->done); + assert(!*b->done); + size_t bufsize = 16ul * 1024ul * 1024ul; + char *buf = malloc(bufsize); + size_t remaining_to_read = b->size_to_read; + size_t remaining_to_write = b->size; + while (remaining_to_read) { + assert(remaining_to_write <= remaining_to_read); + size_t read_now = smin(remaining_to_read, bufsize); + size_t r = fread(buf, 1, read_now, b->input_file); + assert(r <= read_now); + remaining_to_read -= read_now; + if (r < read_now) { + if (feof(b->input_file)) { + fprintf(stderr, "read from %s encountered premature end of file", b->fullname); + spew(SPEW_ERROR, "read from %s encountered premature end of file", b->fullname); + } else { + assert(ferror(b->input_file)); + fprintf(stderr, "read from %s failed with error: %d\n", b->fullname, ferror(b->input_file)); + spew(SPEW_ERROR, "read from %s failed with error: %d\n", b->fullname, ferror(b->input_file)); + } + exit(1); + } else { + size_t write_now = smin(remaining_to_write, read_now); + size_t r2 = fwrite(buf, 1, write_now, of); + assert(r2 == write_now); + remaining_to_write -= write_now; + } + } + FREE(buf); + } + { + int r __attribute__((unused)) = + fchmod(fileno(of), (mode_t)b->mode); + // Don't check r. Either it worked or it didn't. + } + if (arg_preserve) { + int r __attribute__((unused)) = + fchown(fileno(of), (uid_t)b->uid, (gid_t)b->gid); + // Don't check r. Either it worked or it didn't. Gnu + // tar will try doing fchownat as a backup. Maybe + // we'll have to do that too... + } + do_fclose(of); + { + struct timespec times[2] = {{.tv_nsec=UTIME_OMIT}, + {.tv_sec = (time_t)b->mtime, .tv_nsec=0}}; + int r = utimensat(AT_FDCWD, b->fullname, times, AT_SYMLINK_NOFOLLOW); + if (r != 0) { + fprintf(stderr, "utimensat failed on %s errno=%d (%s)\n", + b->fullname, errno, strerror(errno)); + spew(SPEW_ERROR, "utimensat failed on %s errno=%d (%s)", + b->fullname, errno, strerror(errno)); + __sync_add_and_fetch(&some_errors, 1); + } + } + if (!b->bytes) { + *b->done = 1; + spew(SPEW_DEBUG, "Finished immediate read"); + pthread_mutex_lock(&fwmutex); + pthread_cond_signal(&fwcond); + pthread_mutex_unlock(&fwmutex); + } + spew(SPEW_DEBUG, "finished %s", b->fullname); + FREE(b->fullname); + assert(!b->linkname); + FREE(b->bytes); + FREE(b); + break; + } + case '1': { + assert(b->fullname && b->fullname[0]); + assert(b->linkname && b->linkname[0]); + ret = do_link(b->linkname, b->fullname); + if (ret != 0 && retry == 1) { + check_and_delete_conflicts(b->command, b->fullname); + process_fwobject(threadn, b, handle, 0); + return; + } + FREE(b->linkname); + FREE(b->fullname); + FREE(b); + break; + } + case '2': { + ret = do_symlink(b->linkname, b->fullname); + if (ret != 0 && errno == EEXIST && retry == 1) { + check_and_delete_conflicts(b->command, b->fullname); + process_fwobject(threadn, b, handle, 0); + return; + } + { + struct timespec times[2] = {{.tv_nsec=UTIME_OMIT}, + {.tv_sec = (time_t)b->mtime, .tv_nsec=0}}; + int r = utimensat(AT_FDCWD, b->fullname, times, AT_SYMLINK_NOFOLLOW); + assert(r == 0); + } + FREE(b->linkname); + FREE(b->fullname); + FREE(b); + break; + } + case '5': { + ret = do_mkdir(b->fullname); + if (ret != 0 && retry == 1) { + check_and_delete_conflicts(b->command, b->fullname); + process_fwobject(threadn, b, handle, 0); + return; + } + FREE(b->fullname); + assert(!b->linkname); + FREE(b); + break; + } + case '6': { + ret = do_mkfifo(b->fullname, (mode_t)b->mode); + if (ret != 0 && retry == 1) { + check_and_delete_conflicts(b->command, b->fullname); + process_fwobject(threadn, b, handle, 0); + return; + } + FREE(b->fullname); + assert(!b->linkname); + FREE(b); + break; + } + default: { + fprintf(stderr, "Unknown command %c\n", b->command); + spew(SPEW_DEBUG, "Unknown command %c", b->command); + assert(0); + } + } + spew(SPEW_DEBUG, "thread %lu finishing an fwobject", threadn); + fwg_finish_node(fwg, handle); + spew(SPEW_DEBUG, "%lu: next", threadn); +} + +static void process_fwobjects(size_t threadn) { + while (1) { + struct fwobject *b; + struct fwgnode *handle; + + int r = fwg_get_ready_node(fwg, &b, &handle); + if (r != 0) { + return; + } + assert(b); + process_fwobject(threadn, b, handle, 1); + } +} + +static void* fwobjects_runner(void *arg) { + size_t *ip = arg; + process_fwobjects(*ip); + return NULL; +} + +enum { default_n_threads = 32 }; +static size_t n_threads = default_n_threads; +static pthread_t *threads; +static size_t *threadid; + +static void start_runners(void) { + MALLOC_N(threads, n_threads); + MALLOC_N(threadid, n_threads); + for (size_t i = 0; i < n_threads; i++) { + threadid[i]=i; + pthread_create(&threads[i], NULL, fwobjects_runner, &threadid[i]); + } +} +static void runners_finish(void) { + // After the last enqueue, call sfq_finish_up to wait for all the work to finish. + for (size_t i = 0; i < n_threads; i++) { + void *v; + pthread_join(threads[i], &v); + assert(v == NULL); + } + FREE(threads); + FREE(threadid); +} + +static size_t parse_octal_number(char *field, size_t sizeof_field) { + assert(sizeof_field > 0); + if (*field == '\200') { + size_t result = 0; + for (size_t i = 1; i < sizeof_field; i++) { + result = (result * 256) + ((unsigned char *)field)[i]; + } + return result; + } else { + assert(0 == (*(unsigned char*)field & 0x80)); + errno = 0; + char *end; + size_t result = strtoul(field, &end, 8); + assert(errno == 0); + assert(*end == 0 || *end==' '); + assert(end < field+sizeof_field); + return result; + } +} + +static const char *cmd; +static char tarmode = ' '; + +void help(int exitcode) { + spew(SPEW_DEBUG, "exitcode %d ", exitcode); + FILE *out = exitcode ? stderr : stdout; + fprintf(out, "Usage: %s [TAROPTION] [OPTION] FILE...\n", cmd); + fprintf(out, "%s is a parallel implementation of tar. It can create and extract\n", cmd); + fprintf(out, "tarballs in parallel, which can sometimes provide improved performance.\n"); + fprintf(out, " TAROPTION is only the single-letter tar options with no prefix.\n"); + fprintf(out, " TAROPTION can be\n"); + fprintf(out, " p Preserve permissions (default for superuser).\n"); + fprintf(out, " c Create a tarball. Specify FILE(s).\n"); + fprintf(out, " x Extract files. Don't specify FILE(s). You need exactly one of x or c.\n"); + fprintf(out, " z Filter the archive through gzip(1).\n"); + fprintf(out, " f The next argument names the archive (the input tarball for x, the output for c). A dash `-' means stdin/stdout. Without f, the default is stdin/stdout.\n"); + fprintf(out, " OPTION can be\n"); + fprintf(out, " -C DIR Change to directory DIR\n"); + fprintf(out, " -h, --help Print help.\n"); + fprintf(out, " -P P (capital P) use P-fold parallelism (default %d threads).\n", default_n_threads); + fprintf(out, " -m M Set memory budget to M mebibytes (default %d MiB).\n", default_memory_budget_MiB); + fprintf(out, " --exclude=PATTERN Exclude patterns matching PATTERN, a glob(3)-style wildcard pattern.\n"); + fprintf(out, " -p, --preserve-permissions, --same-permissions\n"); + fprintf(out, " extract information about file permissions (default for superuser)\n"); + fprintf(out, " --version Print program version.\n"); + fprintf(out, " --history Print a brief history of the versions.\n"); + fprintf(out, " --progress Monitor progress of untarring.\n"); + fprintf(out, "This is version %s gitrev %s compiled %s %s.\n", + VERSION, gitrev, __DATE__, __TIME__); + fprintf(out, "Report bugs to parallel-tools-support_ww_grp@oracle.com\n"); +} + +static int prefix_matches(const char *prefix, const char *string, char const **suffix) { + while (*prefix == *string) { + spew(SPEW_DEBUG, "prefix %s string %s", prefix, string); + prefix++; string++; + } + if (*prefix == 0) { + *suffix = string; + spew(SPEW_DEBUG, "prefix %s suffix %s string %s", prefix, *suffix, string); + return 1; + } else { + return 0; + } +} + +static int parse_args(int argc, char *argv[], + struct vector_of_strings *vs) { + assert(argc > 0); + cmd = argv[0]; + argc--; argv++; + int at_first_arg = 1; + char const *suffix; + while (argc) { + spew(SPEW_DEBUG, "at_first=%d arg parse = %s", at_first_arg, argv[0]); + if (strcmp(argv[0], "-P") == 0) { + argc--; argv++; + if (argc == 0) { + help(1); + return ERROR; + } + n_threads = parse_number_or_help(argv[0]); + if (n_threads == 0) { + // something is wrong should've got a value + return ERROR; + } + spew(SPEW_DEBUG, "number of threads = %ld", n_threads); + } else if (strcmp(argv[0], "-m") == 0) { + argc--; argv++; + if (argc == 0) { + help(1); + return ERROR; + } + memory_budget_MiB = parse_number_or_help(argv[0]); + if (memory_budget_MiB == 0) { + // something is wrong should've got a value + return ERROR; + } + spew(SPEW_DEBUG, "memory budget (MiB) = %ld", memory_budget_MiB); + } else if (strcmp(argv[0], "-h") == 0 || + strcmp(argv[0], "--help") == 0) { + help(0); + return DONE; + + } else if (strcmp(argv[0], "--version") == 0) { + printf("%s %s\n" + "Copyright (C) 2019 Oracle.\n" + "Written by Bradley C. Kuszmaul\n", + cmd, VERSION); + return DONE; + } else if (strcmp(argv[0], "--history") == 0) { + fprintf(stderr, "%s", VERSION_HISTORY); + return DONE; + } else if (strcmp(argv[0], "-C") == 0) { + argc--; argv++; + if (argc == 0) { + fprintf(stderr, "Need a DIR argument for -C\n"); + spew(SPEW_ERROR, "Need a DIR argument for -C"); + help(1); + return ERROR; + } + arg_change_dir = argv[0]; + } else if (strcmp(argv[0], "--progress") == 0) { + arg_show_progress = 1; + } else if (strcmp(argv[0], "--exclude") == 0) { + argc--; argv++; + if (argc == 0) { + fprintf(stderr, "Need a PATTERN argument for --exclude\n"); + spew(SPEW_ERROR, "Need a PATTERN argument for --exclude"); + help(1); + return ERROR; + } + spew(SPEW_DEBUG, "excluding %s", argv[0]); + vector_of_strings_push(excludes, argv[0]); + } else if (strcmp(argv[0], "-p") == 0 + || strcmp(argv[0], "--preserve-permissions") == 0 + || strcmp(argv[0], "--same-permissions") == 0) { + arg_preserve = 1; + } else if (prefix_matches("--exclude=", argv[0], &suffix)) { + spew(SPEW_DEBUG, "excluding %s suffix %s", argv[0], suffix); + vector_of_strings_push(excludes, suffix); + } else if (at_first_arg && argv[0][0] != '-') { + // tar-type arguments, we accept pxzf + spew(SPEW_DEBUG, "at_first=%d argv[0][0]=%c", at_first_arg, argv[0][0]); + const char *arg = argv[0]; + int has_named_input = 0; + while (arg[0]) { + spew(SPEW_DEBUG, "arg=%s", arg); + switch (arg[0]) { + case 'p': + arg_preserve = 1; + break; + case 'c': + case 'x': + if (tarmode != ' ') { + fprintf(stderr, "incompatible: '%c' and '%c'\n", + tarmode, arg[0]); + spew(SPEW_ERROR, "incompatible: '%c' and '%c", + tarmode, arg[0]); + help(1); + return ERROR; + } + tarmode = arg[0]; + break; // we can only extract + case 'z': + arg_gunzip = 1; + break; + case 'f': + has_named_input = 1; + break; + default: + fprintf(stderr, "Unknown option %c\n", arg[0]); + spew(SPEW_ERROR, "Unknown option %c", arg[0]); + help(1); + return ERROR; + break; + } + arg++; + } + if (has_named_input) { + argc--; argv++; + if (argc == 0) { + fprintf(stderr, "Need a FILE argument for f\n"); + spew(SPEW_DEBUG, "Need a FILE argument for f"); + help(1); + return ERROR; + } + if (strcmp(argv[0], "-") != 0) { + arg_named_input = argv[0]; + } + } + } else { + vector_of_strings_push(vs, argv[0]); + } + argc--; argv++; + at_first_arg = 0; + } + if (tarmode == ' ') { + fprintf(stderr, "You need to specify exactly one of x or c.\n"); + spew(SPEW_ERROR, "You need to specify exactly one of x or c."); + help(1); + return 1; + } + if (tarmode != 'c' && vector_of_strings_size(vs) > 0) { + fprintf(stderr, "Don't specify files except when specifying c.\n"); + spew(SPEW_DEBUG, "Don't specify files except when specifying c."); + help(1); + return ERROR; + } + return CONTINUE; +} + +#define NAMELEN 600 + +// After doing everything, directories may need their permission and +// owner set. We need to do this from the bottom up in the directory +// hierarchy, since if we change the permissions on the root of the +// hierarchy, we may be unable to modify anything else. We'll do it +// in waves, based on the number of slashes in the directory name. + +struct vecitem { // A directory to fix up. + char *fullname; + size_t mode; + size_t uid; + size_t gid; + size_t mtime; +}; + +// Protected by the fixup_mutex +static size_t fixup_slash_count; // the size being worked on right now +static size_t n_fixing_up = 0; // How many are working on fixup slash count. +static struct vec *fixup_dirs; + +static pthread_mutex_t fixup_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t fixup_cond = PTHREAD_COND_INITIALIZER; + +static size_t countslash(const char *s) { + size_t count = 0; + while (*s) { + if ((*s) == '/') + count++; + s++; + } + return count; +} + +static int compare_number_of_slashes(const struct vecitem *a, const struct vecitem *b) { + size_t as = countslash(a->fullname); + size_t bs = countslash(b->fullname); + if (as < bs) { + return -1; + } + if (as > bs) { + return +1; + } + return 0; +} + +static void* fixup_runner(void *ignore __attribute__((__unused__))) { + pthread_mutex_lock(&fixup_mutex); + while (1) { + while (1 + && vec_size(fixup_dirs) + && n_fixing_up + && countslash(vec_peek(fixup_dirs)->fullname) < fixup_slash_count) { + // If n_dirs_to_fixup becomes zero, then the + // fixup_slash_count will also have changed (and a + // broadcast happens when the fixup_slash_count + // changes). + // + // If n_fixing_up becomes zero then we really only care if + // the fixup_slash_count changes (in whichcase a broadcast + // happened) or the n_dirs_to_fixup goes to 0 (in which + // case we broadcast). + // + // If the fixup_slash_count changes we broadcast. + pthread_cond_wait(&fixup_cond, &fixup_mutex); + } + if (vec_size(fixup_dirs) == 0) { + break; // No more work to do + } + size_t sc = countslash(vec_peek(fixup_dirs)->fullname); + if (sc == fixup_slash_count) { + n_fixing_up++; + struct vecitem *dtf = vec_pop(fixup_dirs); + spew(SPEW_DEBUG, "working on %s", dtf->fullname); + pthread_mutex_unlock(&fixup_mutex); + if (arg_preserve && dtf->uid != SIZE_MAX) { + int r __attribute__((unused)) = + chown(dtf->fullname, (uid_t)dtf->uid, (gid_t)dtf->gid); + // Don't complain if this goes wrong + } + { + int r __attribute__((unused)) = + chmod(dtf->fullname, (mode_t)dtf->mode); + // Don't complain if this goes wrong + } + if (dtf->mtime) { + struct timespec times[2] = {{.tv_nsec=UTIME_OMIT}, + {.tv_sec = (time_t)dtf->mtime, .tv_nsec=0}}; + int r = utimensat(AT_FDCWD, dtf->fullname, times, AT_SYMLINK_NOFOLLOW); + assert(r == 0); + } + FREE(dtf->fullname); + FREE(dtf); + pthread_mutex_lock(&fixup_mutex); + n_fixing_up--; + if (n_fixing_up == 0 && vec_size(fixup_dirs) == 0) { + pthread_cond_broadcast(&fixup_cond); + } + } else { + assert(n_fixing_up == 0); + fixup_slash_count = sc; + { // debug code + size_t count = 0; + for (size_t i = 0; i < vec_size(fixup_dirs); i++) { + if (countslash(vec_fetch(fixup_dirs, i)->fullname) == sc) count++; + } + spew(SPEW_DEBUG, "sc=%ld (count=%lu)", sc, count); + } + pthread_cond_broadcast(&fixup_cond); + } + } + pthread_mutex_unlock(&fixup_mutex); + return NULL; +} + +static void Close(int fd) { + int r = close(fd); + assert(r == 0); +} + +static void Dup2(int oldfd, int newfd) { + int r = dup2(oldfd, newfd); + if (r != newfd) { + fprintf(stderr, "dup2(%d %d) -> %d --> errno=%d (%s)\n", + oldfd, newfd, r, errno, strerror(errno)); + spew(SPEW_DEBUG, "dup2(%d %d) -> %d --> errno=%d (%s)", + oldfd, newfd, r, errno, strerror(errno)); + } +} + + +enum { READ_END = 0, WRITE_END = 1 }; + +static int Pipefrom(int fd, const char *execarg, ...) { + size_t n = 0; + char **MALLOC_N(argv, 1); + va_list ap; + va_start(ap, execarg); + while (1) { + REALLOC(argv, n+1); + char *s = va_arg(ap, char *); + argv[n++] = s; + if (s == NULL) break; + } + va_end(ap); + int pipefd[2]; + { + int r = pipe(pipefd); + assert(r == 0); + } + if (fork() == 0) { + Dup2(fd, STDIN_FILENO); + Close(fd); + Dup2(pipefd[WRITE_END], STDOUT_FILENO); + Close(pipefd[WRITE_END]); + Close(pipefd[READ_END]); + execvp(execarg, argv); + assert(0); + } else { + Close(fd); + Close(pipefd[WRITE_END]); + FREE(argv); + return pipefd[READ_END]; + } +} + +static int Pipeto(int fd, const char *execarg, ...) { + size_t n = 0; + char **MALLOC_N(argv, n); + va_list ap; + va_start(ap, execarg); + while (1) { + REALLOC(argv, n+1); + char *s = va_arg(ap, char *); + argv[n++] = s; + if (s == NULL) break; + } + va_end(ap); + int pipefd[2]; + { + int r = pipe(pipefd); + assert(r == 0); + } + if (fork() == 0) { + Dup2(fd, STDOUT_FILENO); + Close(fd); + Dup2(pipefd[READ_END], STDIN_FILENO); + Close(pipefd[WRITE_END]); + Close(pipefd[READ_END]); + execvp(execarg, argv); + assert(0); + } else { + Close(fd); + Close(pipefd[READ_END]); + return pipefd[WRITE_END]; + } +} + +struct stringset *directories_i_know_about; + +static void dirname_of_pathname(const char *pathname, char *dirname) { + // find the destination directory name from the full pathname + size_t filelen = strlen(pathname) + 1; + strncpy(dirname, pathname, filelen); + const size_t nl = strlen(dirname); + if (nl != 0 + && dirname[nl-1] == '/') + { + dirname[nl-1] = 0; + } + char *lastslash = strrchr(dirname, '/'); + if (lastslash) { + *(lastslash+1) = 0; + } else { + dirname[0] = 0; // no slashes, so use the empty dir + } +} + +static void ensure_directory_exists(const char *dirname, int implicitly_create) +// implicitly_create is 1 for directories that are not explicitly created, +// so we should simply mkdir using the default permissions implied by +// whatever umask is. +{ + if (dirname[0] && + !stringset_contains(directories_i_know_about, dirname) != 0) + { + // Don't know about it. + size_t dirlen = strlen(dirname)+1; + char *parentdir = malloc(dirlen); + memset(&parentdir[0], 0, dirlen); + dirname_of_pathname(dirname, parentdir); + ensure_directory_exists(parentdir, 1); + if (implicitly_create) { + int r = mkdir(dirname, 0777); + if (r != 0 && errno != EEXIST) { + fprintf(stderr, "Trying to mkdir %s\n", dirname); + perror("mkdir"); + fprintf(stderr, "I'll try to continue\n"); + spew(SPEW_ERROR, "mkdir %s failed errno=%d (%s)", dirname, errno, strerror(errno)); + } + } else { + struct fwobject *b = mk_fwobject('5', dirname, NULL, 0, 0, 0, 0, 0, NULL); + fwg_add2(fwg, dirname, b, sizeof(*b) + strlen(dirname), + dirname, parentdir); + } + stringset_insert(directories_i_know_about, dirname); + if(parentdir) { + FREE(parentdir); + } + } +} + +static size_t extracttar(void) +// Return the error count. +{ + size_t errcount = 0; + int input_fd = arg_named_input + ? open(arg_named_input, O_RDONLY | O_CLOEXEC) + : dup(STDIN_FILENO); + if(input_fd < 0) { + fprintf(stderr, "No such tar file exists : %s\n", arg_named_input); + errcount++; + return errcount; + } + + int pv_fd = arg_show_progress + ? Pipefrom(input_fd, "pv", "pv", NULL) + : input_fd; + assert(pv_fd >= 0); + + int unzipped_input = arg_gunzip + ? Pipefrom(pv_fd, "gunzip", "gunzip", "-c", NULL) + : pv_fd; + assert(unzipped_input); + + FILE *input_file = fdopen(unzipped_input, "r"); + + if (arg_change_dir) { + int r = chdir(arg_change_dir); + if (r != 0) { + fprintf(stderr, "Couldn't change to dir %s\n", arg_change_dir); + perror("chdir"); + spew(SPEW_ERROR, "chdir %s failed errno=%d (%s)", arg_change_dir, errno, strerror(errno)); + exit(1); + } + } + + fixup_dirs = mk_vec(); + directories_i_know_about = mk_stringset(); + pinit(); + start_runners(); + int saw_end = 0; + size_t count = 0; + + char prev_prev_typeflag = 0; + char prev_typeflag = 0; + + // For long names the encoding is to say (with 'L') "next one or more + // 512-byte block contains the name of the file for the following + // record". Similarly to say with 'K' that the next one or more 512-byte + // block contains the linkname for the file in the following + // record". And if there is an L followed by a K then they both + // modify the following record. + // Here the long_name_from_prev is supposed to store the full name of + // the file for encoding = 'L' which can span across multiple blocks. + + int has_name_from_prev = 0; + char *long_name_from_prev = NULL; + int has_linkname_from_prev = 0; + struct posix_header linkname_from_prev; + linkname_from_prev.name[0] = 0; + + size_t regcount = 0, linkcount = 0, symlinkcount = 0, dircount = 0; + + size_t n_records_seen = 0; + + while (1) { + + if (prev_typeflag != 'L' && prev_typeflag != 'K') { + has_name_from_prev = 0; + has_linkname_from_prev = 0; + } + + char *fullname; + char linkname[600]; + struct posix_header h; + { + static struct posix_header zero; + h = zero; + size_t r = fread(&h, 1, sizeof(h), input_file); + if (r == 0 && feof(input_file)) { + break; + } + if (n_records_seen == 0 && + memcmp(h.magic, "ustar", 5) != 0) + { + fprintf(stderr, "Input does not appear to be a tar file.\n"); + spew(SPEW_ERROR, "Input does not appear to be a tar file."); + if (0 == memcmp(h.name, + (unsigned char[]){0x1f, 0x8b, 0x08}, 3)) + { + fprintf(stderr, "Input appears to be gzipped, "); + spew(SPEW_ERROR, "Input appears to be gzipped, "); + if (arg_gunzip) { + fprintf(stderr, "which is confusing since you specified -z\n"); + spew(SPEW_ERROR, "which is confusing since you specified -z"); + } else { + fprintf(stderr, "without specifying -z, e.g., as\n" + " %s xzf file.tar.gz\n", cmd); + spew(SPEW_ERROR, "without specifying -z, e.g., as\n" + " %s xzf file.tar.gz", cmd); + } + } else if (0 == + memcmp( + h.name, + (unsigned char[]){0xFD, '7', 'z', 'X', 'Z'}, + 5)) { + fprintf(stderr, + "Input appears to be xz compressed," + " which %s cannot (yet) decompress.\n", + cmd); + fprintf(stderr, + "Maybe you can use an external decompressor," + " e.g., as\n xzcat file.tar.xz|%s xf -\n", + cmd); + spew(SPEW_ERROR, + "Input appears to be xz compressed," + " which %s cannot (yet) decompress.", + cmd); + spew(SPEW_ERROR, + "Maybe you can use an external decompressor," + " e.g., as\n xzcat file.tar.xz|%s xf -", + cmd); + } else { + fprintf(stderr, "Maybe it needs to be decompressed.\n"); + spew(SPEW_DEBUG, "Maybe it needs to be decompressed."); + } + exit(1); + } + assert(r == sizeof(h)); + n_records_seen++; + } + count++; + { + char *hname = h.name; + size_t hname_limit = sizeof(h.name); + if (has_name_from_prev) { + hname = long_name_from_prev; + spew(SPEW_DEBUG, "prev record name = %s", hname); + hname_limit = strlen(long_name_from_prev); + } else { + if(long_name_from_prev) { + FREE(long_name_from_prev); + } + } + // The tar docs say that linkname is NUL terminated, but it's not. + if (has_linkname_from_prev) { + snprintf(linkname, sizeof(linkname), "%.*s", + (int)512, linkname_from_prev.name); + spew(SPEW_DEBUG, "p linkname=%s (%ld)", linkname, sizeof(linkname_from_prev.name)); + } else { + snprintf(linkname, sizeof(linkname), "%.*s", + (int)sizeof(h.linkname), h.linkname); + spew(SPEW_DEBUG, "h linkname=%s", linkname); + } + if (h.prefix[0]) { + fullname = malloc(hname_limit + sizeof(h.prefix) + 1); + int flen = snprintf(fullname, hname_limit + sizeof(h.prefix) + 1, "%.*s/%.*s", + (int)sizeof(h.prefix), h.prefix, + (int)hname_limit, hname); + assert((size_t)flen <= (hname_limit + sizeof(h.prefix))); + } else { + fullname = malloc(hname_limit + 1); + int flen = snprintf(fullname, hname_limit + 1, "%.*s", + (int)hname_limit, hname); + assert((size_t)flen <= hname_limit); + } + spew(SPEW_DEBUG, "prefix=%s fullname=%s", h.prefix, fullname); + } + + int exclude_me = matches_excludes(fullname); + spew(SPEW_DEBUG, "Exclude %s --> %d", fullname, exclude_me); + + size_t dirlen = strlen(fullname)+1; + char *dirname = malloc(dirlen); + memset(dirname, 0, strlen(fullname)); + dirname_of_pathname(fullname, dirname); + spew(SPEW_DEBUG, "dirname=%s sizeof(dirname)=%lu fullname=%s", + dirname, sizeof(dirname), fullname); + if (!exclude_me) + ensure_directory_exists(dirname, 1); + + assert(h.pad[0]==0); + size_t size_to_write = parse_octal_number(h.size, sizeof(h.size)); + size_t size_to_read = ((size_to_write+511)/512)*512; + if (size_to_write) { + spew(SPEW_DEBUG, "Write size=%lu", size_to_write); + } + if (size_to_read) { + spew(SPEW_DEBUG, "Read size=%lu ", size_to_read); + } + spew(SPEW_DEBUG, "type=%c", *h.typeflag); + + size_t mode = parse_octal_number(h.mode, sizeof(h.mode)); + + if (*h.typeflag == '5') { + // Mkdir. + if (!exclude_me) { + dircount++; + ensure_directory_exists(fullname, 0); + struct vecitem *MALLOC(dtf); + *dtf = (struct vecitem){ + strdup(fullname), + mode, + parse_octal_number(h.uid, sizeof(h.uid)), + parse_octal_number(h.gid, sizeof(h.gid)), + parse_octal_number(h.mtime, sizeof(h.mtime)) + }; + vec_push(fixup_dirs, dtf); + } + } else if (*h.typeflag == '0') { + regcount++; + // Regular file. + + spew(SPEW_DEBUG, "insert regular file %s size=%lu mode=0%lo", + fullname, size_to_write, mode); + if (exclude_me) { + size_t bufsize = 16ul * 1024ul * 1024ul; + char *bytes = malloc(bufsize); + while (size_to_read > 0) { + size_t read_now = smin(size_to_read, bufsize); + size_t s = do_read(input_file, bytes, read_now); + if (s != read_now) { + fprintf(stderr, "Tried to read %ld bytes, got %ld\n", read_now, s); + spew(SPEW_WARN, "Tried to read %ld bytes, got %ld", read_now, s); + } + assert(s == read_now); + size_to_read -= read_now; + } + free(bytes); + } else if (size_to_read * 2 > memory_budget_MiB * MiB) { + int done = 0; + struct fwobject *b = mk_fwobject_read_from_input( + *h.typeflag, + fullname, + size_to_write, + mode, + parse_octal_number(h.uid, sizeof(h.uid)), + parse_octal_number(h.gid, sizeof(h.gid)), + parse_octal_number(h.mtime, sizeof(h.mtime)), + input_file, + size_to_read, + &done); + fwg_add2(fwg, fullname, + b, sizeof(*b) + strlen(fullname), + fullname, dirname); + pthread_mutex_lock(&fwmutex); + while (!done) { + pthread_cond_wait(&fwcond, &fwmutex); + } + pthread_mutex_unlock(&fwmutex); + spew(SPEW_DEBUG, "main proceeding"); + } else { + // malloc and free are fine with size_to_read==0. + char *bytes = malloc(size_to_read); + spew(SPEW_DEBUG, "malloc(%zu) = %p", size_to_read, bytes); + if (size_to_read) { + size_t s = do_read(input_file, bytes, size_to_read); + assert(s == size_to_read); + } + struct fwobject *b = mk_fwobject(*h.typeflag, + fullname, + NULL, + size_to_write, + mode, + parse_octal_number(h.uid, sizeof(h.uid)), + parse_octal_number(h.gid, sizeof(h.gid)), + parse_octal_number(h.mtime, sizeof(h.mtime)), + bytes); + fwg_add2(fwg, fullname, + b, sizeof(*b) + strlen(fullname) + size_to_write, + fullname, dirname); + } + } else if (*h.typeflag == 0) { + if (saw_end == 0) { + saw_end = 1; + } else { + if (dirname) { + FREE(dirname); + } + if (fullname) { + FREE(fullname); + } + break; + } + } else if (*h.typeflag == 'g') { + spew(SPEW_DEBUG, "global extended header"); + char *bytes = malloc(size_to_read); + assert(bytes); + size_t s = do_read(input_file, bytes, size_to_read); + assert(s == size_to_read); + free(bytes); + } else if (*h.typeflag == '1') { + // We have to wait on the fullname and the linkname. It's + // possible that the directories are the same. + linkcount++; + assert(linkname[0] && fullname[0]); + // If the fullname and the linkname are the same, then + // don't do anything. That might not be right if the new + // link has different permissions or something. + if (!exclude_me && + strcmp(fullname, linkname) != 0) + { + struct fwobject *b = mk_fwobject(*h.typeflag, + fullname, + linkname, + 0, 0, 0, 0, 0, NULL); + spew(SPEW_DEBUG, "adding '%c'", *h.typeflag); + fwg_add3(fwg, fullname, + b, sizeof(*b) + strlen(fullname) + strlen(linkname), + fullname, dirname, linkname); + } + } else if (!exclude_me && + (*h.typeflag == '2' // symlink + || *h.typeflag == '6' // fifo + )) { + if (*h.typeflag == 2) symlinkcount++; + struct fwobject *b = mk_fwobject(*h.typeflag, + fullname, + *h.typeflag == '2' ? linkname : NULL, + 0, + mode, 0, 0, 0, + NULL); + spew(SPEW_DEBUG, "adding '%c'", *h.typeflag); + fwg_add2(fwg, fullname, b, sizeof(*b) + strlen(fullname) + strlen(linkname), + fullname, dirname); + } else if (*h.typeflag == 'L') { + if (0 && prev_typeflag != '5' && prev_typeflag != '0') { + fprintf(stderr, "L: prev_typeflag = %c\n", prev_typeflag); + spew(SPEW_DEBUG, "L: prev_typeflag = %c", prev_typeflag); + } + long_name_from_prev = malloc(size_to_read); + size_t r = fread(long_name_from_prev, 1, size_to_read, input_file); + assert(r == size_to_read); + spew(SPEW_DEBUG, "Long name: name=%s\n", long_name_from_prev); + has_name_from_prev = 1; + } else if (*h.typeflag == 'K') { + if (0 && prev_typeflag != '5' && prev_typeflag != '0') { + fprintf(stderr, "K: prev_typeflag = %c\n", prev_typeflag); + spew(SPEW_DEBUG, "K: prev_typeflag = %c", prev_typeflag); + } + size_t r = fread(&linkname_from_prev, 1, sizeof(linkname_from_prev), input_file); + assert(r == sizeof(linkname_from_prev)); + has_linkname_from_prev = 1; + } else { + fprintf(stderr, "h.typeflag=%c (prev_typeflag=%c) (prev_prev_typeflag=%c)\n", *h.typeflag, prev_typeflag, prev_prev_typeflag); + spew(SPEW_DEBUG, "h.typeflag=%c (prev_typeflag=%c) (prev_prev_typeflag=%c)", *h.typeflag, prev_typeflag, prev_prev_typeflag); + debug_print_posix_header(&h); + abort(); + } + prev_prev_typeflag = prev_typeflag; + prev_typeflag = *h.typeflag; + if (dirname) { + FREE(dirname); + } + if (fullname) { + FREE(fullname); + } + } + fwg_end_of_nodes(fwg); + runners_finish(); + if (vec_size(fixup_dirs)) { + const size_t time_fixup = 0; + struct timespec start_fixup, end_fixup; + if (time_fixup) { + clock_gettime(CLOCK_MONOTONIC, &start_fixup); + fprintf(stderr, "Fixing up, there are %lu dirs\n", vec_size(fixup_dirs)); + spew(SPEW_DEBUG, "Fixing up, there are %lu dirs", vec_size(fixup_dirs)); + } + vec_sort(fixup_dirs, compare_number_of_slashes); + fixup_slash_count = countslash(vec_peek(fixup_dirs)->fullname); + pthread_t *MALLOC_N(fthreads, n_threads); + for (size_t i = 0; i < n_threads; i++) { + pthread_create(&fthreads[i], NULL, fixup_runner, NULL); + } + for (size_t i = 0; i < n_threads; i++) { + void *v; + pthread_join(fthreads[i], &v); + } + FREE(fthreads); + if (time_fixup) { + clock_gettime(CLOCK_MONOTONIC, &end_fixup);; + fprintf(stderr, "Fixup took %6.3fs\n", (double)(end_fixup.tv_sec - start_fixup.tv_sec) + 1e-9*(double)(end_fixup.tv_nsec - start_fixup.tv_nsec)); + spew(SPEW_DEBUG, "Fixup took %6.3fs", (double)(end_fixup.tv_sec - start_fixup.tv_sec) + 1e-9*(double)(end_fixup.tv_nsec - start_fixup.tv_nsec)); + } + } + // In some cases the dirs_to_fixup are still there (e.g., if we aren't preserving) + while (1) { + struct vecitem *item = vec_pop(fixup_dirs); + if (item == NULL) { + break; + } + FREE(item->fullname); + FREE(item); + } + pdestroy(); + fclose(input_file); + directories_i_know_about = stringset_destroy(directories_i_know_about); + fixup_dirs = vec_destroy(fixup_dirs); + + spew(SPEW_DEBUG, "regcount=%ld linkcount=%ld symlinkcount=%ld dircount=%lu", regcount, linkcount, symlinkcount, dircount); + while (1) { + int status; + int r = wait(&status); + if (r == -1 && errno == ECHILD) { + break; + } + if (r == -1) { + fprintf(stderr, "%s: waiting on child %s\n", cmd, strerror(errno)); + spew(SPEW_DEBUG, "%s: waiting on child %s", cmd, strerror(errno)); + errcount++; + break; + } + if (!WIFEXITED(status)) { + fprintf(stderr, "%s: Child did not return normally\n", cmd); + spew(SPEW_ERROR, "%s: Child did not return normally", cmd); + errcount++; + } else if (WEXITSTATUS(status) != 0) { + fprintf(stderr, "%s: Child returned status %d\n", cmd, WEXITSTATUS(status)); + spew(SPEW_ERROR, "%s: Child returned status %d", cmd, WEXITSTATUS(status)); + errcount++; + } + } + if (errcount == 0 && n_records_seen == 0) { + fprintf(stderr, "%s: This does not look like a tar archive\n", cmd); + spew(SPEW_ERROR, "%s: This does not look like a tar archive", cmd); + errcount++; + } + return errcount; +} + +// For createtar +static FILE *output_file; +static size_t n_written = 0; +static pthread_mutex_t emit_mutex = PTHREAD_MUTEX_INITIALIZER; + +static unsigned char ph_string[512]; +static size_t ph_off; + +static void e_char(unsigned char c) { + ph_string[ph_off++] = c; +} + +static void e_string_short(const char *string, size_t len) { + assert(ph_off + len <= sizeof(ph_string)); + size_t slen = strlen(string); + if(slen < len) { + memcpy(ph_string + ph_off, string, slen); + } + else { + memcpy(ph_string + ph_off, string, len); + } + while (slen < len) { + ph_string[ph_off + slen++] = 0; + } + ph_off += len; +} + +static void e_string(const char *string, size_t len) { + assert(strlen(string) <= len); + e_string_short(string, len); +} + +static void e_number(size_t number, size_t len) { + int r = snprintf((char*)ph_string+ph_off, 512-ph_off, "%.*zo", + (int)len-1, number); + assert(r >= 0); + assert(ph_off + (size_t)r < 512); + if ((size_t)r > len-1) { + // It's too big, and must be represented in binary. + assert((len > sizeof(size_t)) || (0 == (number >> (8 * len - 1)))); + for (size_t i = len; i > 0; i--) { + unsigned char extra_bits = (i == 1) ? 0x80u : 0; + unsigned char the_byte = (unsigned char)(number & 0xffu); + ph_string[ph_off + i - 1] = extra_bits | the_byte; + number >>= 8; + } + } + ph_off += len; +} + +static void e_checksum_blanks(void) { + for (size_t i = 0; i < 8; i++) e_char(' '); +} + + +static void e_prepare_checksum(void) { + size_t checksum = 0; + for (size_t i = 0; i < 512; i++) checksum += (unsigned char)ph_string[i]; + snprintf((char*)ph_string+148, 8, "%06zo", checksum); +} + +static void emit_longlink(const char *fname, unsigned char typ) { + e_string("././@LongLink", 100); + e_number(0644, 8); + e_number(0, 8); + e_number(0, 8); + e_number(strlen(fname)+1, 12); + e_number(0, 12); + e_checksum_blanks(); + e_char(typ); + e_string("", 100); + e_string("ustar ", 8); + e_string("root", 32); + e_string("root", 32); + e_string("", 8+8+155+12); + assert(ph_off == 512); + e_prepare_checksum(); + fwrite(ph_string, 1, ph_off, output_file); + n_written += ph_off; + ph_off = 0; + fputs(fname, output_file); + n_written += strlen(fname); + size_t i = strlen(fname); + if(!(i % 512)) { + putc(0, output_file); + n_written++; + i++; + } + for (; i % 512; i++) { + putc(0, output_file); + n_written++; + } +} + +static void emit_header_locked(const char *fname, + struct stat *statbuf, + const char *link_target, + unsigned char link_typeflag) { + // To get it going, assume that the fname is less than 100 chars + ph_off = 0; + if (link_target) { + if (strlen(link_target) > 100) { + emit_longlink(link_target, 'K'); + } + } + if (S_ISDIR(statbuf->st_mode)) { + char *output_fname; + int r = asprintf(&output_fname, "%s/", fname); + assert(r >= 0); + if (strlen(fname)+1 > 100) { + emit_longlink(output_fname, 'L'); + e_string_short(output_fname, 100); + } else { + e_string(output_fname, 100); + } + free(output_fname); + } else { + if (strlen(fname) > 100) { + emit_longlink(fname, 'L'); + e_string_short(fname, 100); + } else { + e_string(fname, 100); + } + } + e_number(statbuf->st_mode & 07777, 8); + e_number(statbuf->st_uid, 8); + e_number(statbuf->st_gid, 8); + assert(statbuf->st_size >= 0); + if (S_ISLNK(statbuf->st_mode) || S_ISDIR(statbuf->st_mode) || link_target) { + e_number(0, 12); + } else { + e_number((size_t)statbuf->st_size, 12); + } + e_number((size_t)statbuf->st_mtim.tv_sec, 12); + e_checksum_blanks(); + if (link_typeflag) { + e_char(link_typeflag); + } else if (S_ISREG(statbuf->st_mode)) { + e_char('0'); + } else if (S_ISLNK(statbuf->st_mode)) { + e_char('2'); + } else if (S_ISDIR(statbuf->st_mode)) { + e_char('5'); + } else if (S_ISFIFO(statbuf->st_mode)) { + e_char('6'); + } else { + abort(); + } + if (S_ISLNK(statbuf->st_mode)) { + assert(link_target); + if (strlen(link_target) <= 100) { + e_string(link_target, 100); + } else { + e_string_short(link_target, 100); + } + } else { + if (link_target && strlen(link_target) <= 100) { + e_string(link_target, 100); + } else { + e_string("", 100); + } + } + e_string("ustar ", 8); + { // uname + struct passwd *pw = getpwuid(statbuf->st_uid); + if (pw == NULL) { + e_string("", 32); + } else { + e_string(pw->pw_name, 32); + } + } + { // gname + struct group *g = getgrgid(statbuf->st_gid); + if (g == NULL) { + e_string("", 32); + } else { + e_string(g->gr_name, 32); + } + } + e_string("", 8); // devmajor + e_string("", 8); // devminor + e_string("", 155); + e_string("", 12); + assert(ph_off == 512); + e_prepare_checksum(); + fwrite(ph_string, 1, ph_off, output_file); + ph_off = 0; + n_written += 512; +} + +static void emit_data_locked(const char *fname, size_t size) { + FILE *f = fopen(fname, "r"); + if (f == NULL) { + return; + } + const size_t size_limit = 1024*1024u; // bigger than this, we grab the lock early + char *buf=malloc(size_limit); + size_t remaining_size = size; + while (remaining_size) { + size_t read_now = remaining_size > size_limit ? size_limit : remaining_size; + size_t actual_read = fread(buf, 1, read_now, f); + size_t actual_write = fwrite(buf, 1, actual_read, output_file); + n_written += actual_read; + remaining_size -= actual_write; + } + while (n_written % 512) { + n_written++; + putc(' ', output_file); + } + fclose(f); + FREE(buf); +} + +struct inodemap *inset; + +static int maybe_emit_link_locked(const char *fname, struct stat *statbuf) { + void const *value; + if (inodemap_lookup(inset, statbuf->st_ino, statbuf->st_dev, &value)) { + emit_header_locked(fname, statbuf, value, '1'); + return 1; + } else { + inodemap_insert(inset,statbuf->st_ino, statbuf->st_dev, strdup(fname)); + return 0; + } +} + +static void emit(const char *fname, struct stat *statbuf) { + const off_t size_limit = 1024*1024u; // bigger than this, we grab the lock early + if (S_ISREG(statbuf->st_mode)) { + if (statbuf->st_size > size_limit) { + pthread_mutex_lock(&emit_mutex); + if (!maybe_emit_link_locked(fname, statbuf)) { + emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0); + emit_data_locked(fname, (size_t)statbuf->st_size); + } + pthread_mutex_unlock(&emit_mutex); + } else { + assert(statbuf->st_size >= 0); + char *buf = malloc((size_t)statbuf->st_size); + FILE *f = fopen(fname, "r"); + if (f == NULL) { + FREE(buf); + return; + } + size_t s = statbuf->st_size == 0 ? 0 : do_read(f, buf, (size_t)statbuf->st_size); + assert(s == (size_t)statbuf->st_size); + fclose(f); + pthread_mutex_lock(&emit_mutex); + if (!maybe_emit_link_locked(fname, statbuf)) { + emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0); + fwrite(buf, 1, s, output_file); + size_t target = ((s+511)/512)*512; + for (size_t i = s; i < target; i++) { + putc(0, output_file); + } + n_written += target; + } + pthread_mutex_unlock(&emit_mutex); + FREE(buf); + } + } else if (S_ISLNK(statbuf->st_mode)) { + assert(statbuf->st_size >= 0); + char *buf = malloc((size_t)statbuf->st_size + 2); + ssize_t r = readlink(fname, buf, (size_t)statbuf->st_size); + assert(r == statbuf->st_size); + buf[statbuf->st_size] = 0; + pthread_mutex_lock(&emit_mutex); + if (!maybe_emit_link_locked(fname, statbuf)) { + emit_header_locked(fname, statbuf, buf, '2'); + } + pthread_mutex_unlock(&emit_mutex); + FREE(buf); + } else { + pthread_mutex_lock(&emit_mutex); + if (!maybe_emit_link_locked(fname, statbuf)) { + emit_header_locked(fname, statbuf, /*symlink*/NULL, /*linktype*/0); + } + pthread_mutex_unlock(&emit_mutex); + } +} + +static magic_t ctar_magic = "ctar magic"; + +struct ctarframe { + magic_t *magic; + struct vector_of_strings *vs; + size_t inlet_sum; + size_t sum; +}; + +static struct ctarframe *mk_ctarframe(struct vector_of_strings *vs) { + struct ctarframe *MALLOC(cf); + *cf = (struct ctarframe){.magic = &ctar_magic, + .vs = vs, + .inlet_sum = 0, + .sum = 0}; + return cf; +} + +static struct ctarframe *ctarframe_free(struct ctarframe *cf) { + vector_of_strings_destroy(cf->vs); + FREE(cf); + return NULL; +} + +static struct ctarframe *ctarframe_cast(void *app_frame) { + struct ctarframe *cf = app_frame; + assert(cf->magic == &ctar_magic); + return cf; +} + +static void ctarfun_return_inlet(void *parent_app_frame, size_t result) { + struct ctarframe *cf = ctarframe_cast(parent_app_frame); + cf->inlet_sum += result; +} + +static size_t ctar_error_count = 0; +static pthread_mutex_t ctar_error_count_mutex = PTHREAD_MUTEX_INITIALIZER; +static void bump_ctar_error_count(void) { + pthread_mutex_lock(&ctar_error_count_mutex); + ctar_error_count++; + pthread_mutex_unlock(&ctar_error_count_mutex); +} + + +static struct frame *ctarfun2(struct frame *frame, void *app_frame); + +static struct frame*ctarfun(struct frame *frame, void *app_frame) { + struct ctarframe *cf = ctarframe_cast(app_frame); + char *fname; + while ((fname = vector_of_strings_pop(cf->vs))) { + if (!matches_excludes(fname)) { + struct stat statbuf; + { + int r = lstat(fname, &statbuf); + if (r != 0) { + fprintf(stderr, "%s: %s: Cannot stat: %s\n", cmd, fname, strerror(errno)); + spew(SPEW_ERROR, "%s: %s: Cannot stat: %s", cmd, fname, strerror(errno)); + bump_ctar_error_count(); + continue; + } + } + cf->sum++; + emit(fname, &statbuf); + if (S_ISDIR(statbuf.st_mode)) { + struct vector_of_strings *vs = mk_vector_of_strings(); + DIR *dir = opendir(fname); + assert(dir); + struct dirent *entry; + while ((entry = readdir(dir))) { + if (strcmp(entry->d_name, ".") == 0) continue; + if (strcmp(entry->d_name, "..") == 0) continue; + char *path = pathcat(fname, entry->d_name); + vector_of_strings_push(vs, path); + FREE(path); + } + free(fname); + closedir(dir); + struct ctarframe *subframe = mk_ctarframe(vs); + return ctarfun( + sched_spawn(frame, ctarfun, subframe, ctarfun_return_inlet), + subframe); + } + } + free(fname); + } + return sched_sync(frame, ctarfun2); +} + +static struct frame *ctarfun2(struct frame *frame, void *app_frame) { + struct ctarframe *cf = ctarframe_cast(app_frame); + size_t sum = cf->inlet_sum + cf->sum; + cf = ctarframe_free(cf); + return sched_return(frame, sum); +} + +static void do_createtar(struct vector_of_strings *vs) { + struct ctarframe *cf = mk_ctarframe(vs); + size_t n = prun(n_threads, ctarfun, cf); + spew(SPEW_INFO, "Tarred %zu objects", n); +} + +static size_t createtar(struct vector_of_strings *vs) { + ctar_error_count = 0; + int output_fd = arg_named_input + ? open(arg_named_input, O_CREAT | O_TRUNC | O_WRONLY | O_CLOEXEC, 0666) + : dup(STDOUT_FILENO); + assert(output_fd >= 0); + int unzipped_output = arg_gunzip + ? Pipeto(output_fd, "gzip", "gzip", "-c", NULL) + : output_fd; + output_file = fdopen(unzipped_output, "w"); + if (arg_change_dir) { + int r = chdir(arg_change_dir); + if (r != 0) { + fprintf(stderr, "Couldn't change to dir %s\n", arg_change_dir); + perror("chdir"); + spew(SPEW_ERROR, "Couldn't change to dir %s errno=%d (%s)", arg_change_dir, errno, strerror(errno)) ; + exit(1); + } + } + do_createtar(vs); + spew(SPEW_INFO, "n-written=%zu\n", n_written); + while (n_written % (20*512u)) { + // Tar puts at least 2 512-byte blocks at the end, and enough to add up to a multiple of 20 512 blocks. + putc(0, output_file); + n_written++; + } + fclose(output_file); + return ctar_error_count++; +} + +static void destroy_string(void *sv) { + FREE(sv); +} + +int main (int argc, char *argv[]) { + int ret = 0; + find_open_fds_at_start(); + we_are_root = geteuid () == ROOT_UID; + if (we_are_root) arg_preserve = 1; + struct vector_of_strings *vs = mk_vector_of_strings(); + excludes = mk_vector_of_strings(); + +#ifdef SPEW_ENABLED + size_t spew_level = SPEW_DEBUG; + + spew_config(argc, argv, + "partar_log", + /* log roll time in minutes */1440, + /* spew_to_stderr */0, + /*debug=*/0); + spew_set_level(spew_level); + spew(SPEW_INFO, "partar %s, SPLASH!!!", gitrev); +#endif + + ret = parse_args(argc, argv, vs); + switch (ret) { + case DONE: + case ERROR: + goto do_exit; + break; + case CONTINUE: + ret = 0; + break; + default: + ret = 1; + goto do_exit; + break; + } + // parsing done and there is work to do! + spew(SPEW_DEBUG, "arg_change_dir = %s", arg_change_dir); + spew(SPEW_DEBUG, "\targ_named_input = %s", arg_named_input); + + size_t errcount = 0; + if (tarmode == 'x') { + errcount = extracttar(); + vector_of_strings_destroy(vs); + } else if (tarmode == 'c') { + inset = mk_inodemap(); + errcount = createtar(vs); + inset = inodemap_destroy(inset, destroy_string); + } else { + assert(0); + } + + if (errcount || some_errors) { + fprintf(stderr, + "%s (%s %s): Exiting with failure status due to previous errors\n", + cmd, VERSION, gitrev); + spew(SPEW_ERROR, + "%s (%s %s): Exiting with failure status due to previous errors", + cmd, VERSION, gitrev); + ret = 2; + } + +do_exit: + spew_teardown(); + fdleak_check(); + vector_of_strings_destroy(excludes); + return ret; +} diff --git a/software/fss-parallel-tools/putils.c b/software/fss-parallel-tools/putils.c new file mode 100644 index 0000000..4eebbc3 --- /dev/null +++ b/software/fss-parallel-tools/putils.c @@ -0,0 +1,244 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#define _GNU_SOURCE +#include "putils.h" +#include "malloc.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "common/spew.h" + +FILE* do_creat(const char *fullname) { + int fd = open(fullname, O_CREAT | O_WRONLY | O_TRUNC, S_IWUSR); + if (fd == -1) { + if (errno == EEXIST || errno == EISDIR) { + return NULL; + } + fprintf(stderr, "Trying to open %s:\n", fullname); + perror("open"); + spew(SPEW_ERROR, "Create of %s failed errno=%d (%s)", + fullname, errno, strerror(errno)); + abort(); + } + assert(fd >= 0); + FILE *f = fdopen(fd, "w"); + if (!f) { + perror("fdopen"); + spew(SPEW_ERROR, "fdopen of %s for write failed errno=%d (%s)", + fullname, errno, strerror(errno)); + } + assert(f); + return f; +} + +FILE* do_open_for_read(const char *fullname) { + int fd = open(fullname, O_RDONLY); + if (fd == -1) { + fprintf(stderr, "Trying to open %s:\n", fullname); + perror("open"); + spew(SPEW_ERROR, "Open of %s failed errno=%d (%s)", + fullname, errno, strerror(errno)); + abort(); + } + assert(fd >= 0); + FILE *f = fdopen(fd, "r"); + if (!f) { + perror("fdopen"); + spew(SPEW_ERROR, "fdopen of %s for read failed errno=%d (%s)", + fullname, errno, strerror(errno)); + } + assert(f); + return f; +} + + +void do_fclose(FILE *f) { + int r = fclose(f); + if (r != 0) { + perror("fclose"); + spew(SPEW_ERROR, "fclose %p failed errno=%d (%s)", + f, errno, strerror(errno)); + } + assert(r == 0); +} + +void do_write(FILE *f, const char *bytes, size_t size) { + size_t r = fwrite(bytes, 1, size, f); + if (r != size) { + perror("fwrite"); + spew(SPEW_ERROR, "fwrite failed errno=%d (%s)", + errno, strerror(errno)); + } + assert(r == size); +} + +size_t do_read(FILE *f, char *bytes, size_t size) { + // We might be reading from a socket or a pipe, so keep reading till we get all the bytes. + size_t r = fread(bytes, 1, size, f); + if (r == 0) { + if (feof(f)) { + return 0; + } + assert(ferror(f)); + fprintf(stderr, "do_read got error %d\n", ferror(f)); + spew(SPEW_ERROR, "fread failed errno=%d (%s)", + errno, strerror(errno)); + assert(0); + } + if (r < size) { + return r + do_read(f, bytes+r, size-r); + } else { + return r; + } +} + +int do_mkdir(const char *fullname) { + if (0) fprintf(stderr, "mkdir(\"%s\");\n", fullname); + int r = mkdir(fullname, S_IRUSR|S_IWUSR|S_IXUSR); + if (r != 0) { + if (errno == EEXIST) { + return r; + } + // We ensure that the parent exists before we call do_mkdir, + // so ENOENT is a real error. + fprintf(stderr, "Trying to mkdir %s\n", fullname); + perror("mkdir"); + spew(SPEW_ERROR, "mkdir %s failed errno=%d (%s)", + fullname, errno, strerror(errno)); + abort(); + } + assert(r == 0); + return 0; //returns Success +} + +int do_mkfifo(const char *fullname, mode_t mode) { + int r = mkfifo(fullname, mode); + if (r) { + if (errno == EEXIST) { + return r; + } + fprintf(stderr, "Trying to mkfifo(%s, %d)\n", fullname, mode); + perror("mkfifo"); + spew(SPEW_ERROR, "mkfifo %s failed errno=%d (%s)", + fullname, errno, strerror(errno)); + abort(); + } + return 0; //returns Success +} + +int do_link(const char *linkto, const char *fullname) +// Effect create a hardlink. If the file already exists delete it and then +// link. +{ + int r = link(linkto, fullname); + if (r && errno == EEXIST) { + return r; + } + if (r) { + fprintf(stderr, "Trying to link \"%s\" --> \"%s\"\nerrno=%d\n", fullname, linkto, errno); + perror("link"); + spew(SPEW_ERROR, "linking \"%s\" --> \"%s\" failed errno=%d (%s)", + fullname, linkto, errno, strerror(errno)); + abort(); + } + return 0; //returns Success +} + + +int do_symlink(const char *contents, const char *fullname) { + int r = symlink(contents, fullname); + return r; +} + +void do_unlink(const char *fullname) { + int r = unlink(fullname); + assert(r == 0); +} +void do_rmdir(const char *fullname) { + int r = rmdir(fullname); + assert(r == 0); +} + +/* + * This function is callback for nftw call that is made + * with the intention of deleting conflicts. The function + * deletes file or directory provided in the argument 'path' + */ +int delete_subtree(const char *path, const struct stat *st, int typeflag, struct FTW *d_ftw) { + int ret = 1; + if (typeflag == 4 || d_ftw == NULL) { + return ret; + } + if (S_ISDIR(st->st_mode)) { + ret = rmdir(path); + } else { + ret = unlink(path); + } + return ret; +} + +/* Recursively deletes files/directories in directory path passed */ +void delete_dir_recursively(const char *path) { + int ret = nftw(path, delete_subtree, 10, FTW_DEPTH); + if (ret != 0) { + spew(SPEW_ERROR, "Unable to delete subtree for %s. Aborting\n", path); + abort(); + } +} + + +char *pathcat(const char *a, const char *b) { + char *result = NULL; + int r = asprintf(&result, "%s/%s", a, b); + assert(r > 0); + return result; +} + +size_t parse_number_or_help(const char *numstring) { + errno = 0; + char *end; + size_t result = strtoul(numstring, &end, 0); + if (0 + || errno != 0 + || *end != 0) + { + fprintf(stderr, "Trying to parse %s as a number\n", numstring); + spew(SPEW_DEBUG, "Trying to parse %s as a number", numstring); + help(1); + return 0; + } + return result; +} + +void +maybe_error(int r, const char *fun, const char *fname, const char *srcfile, int line) { + if (r != 0) { + fprintf(stderr, "%s:%d Could not %s(\"%s\"). errno=%d (%s)\n", + srcfile, line, fun, fname, errno, strerror(errno)); + spew(SPEW_ERROR, "Could not %s(\"%s\"). errno=%d (%s)", + fun, fname, errno, strerror(errno)); + exit(1); + } +} + +void +maybe_error2(int r, const char *fun, const char *fname, const char *fname2, const char *srcfile, int line) { + if (r != 0) { + fprintf(stderr, "%s:%d Could not %s(\"%s\", \"%s\"). errno=%d (%s)\n", + srcfile, line, + fun, fname, fname2, errno, strerror(errno)); + spew(SPEW_ERROR, "Could not %s(\"%s\", \"%s\"). errno=%d (%s)", + fun, fname, fname2, errno, strerror(errno)); + exit(1); + } +} diff --git a/software/fss-parallel-tools/putils.h b/software/fss-parallel-tools/putils.h new file mode 100644 index 0000000..701063a --- /dev/null +++ b/software/fss-parallel-tools/putils.h @@ -0,0 +1,55 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef PUTILS_H +#define PUTILS_H +#include +#include +FILE* do_creat(const char *fullname); +FILE* do_open_for_read(const char *fullname); +void do_fclose(FILE *f); +void do_write(FILE *f, const char *bytes, size_t size); +size_t do_read(FILE *f, char *bytes, size_t size); + +int do_mkdir(const char *fullname); +// Effect: Create a directory (and its parents). If the directory exists, don't complain. + +int do_mkfifo(const char *fullname, mode_t mode); +// Effect: Create a fifo. + +int do_link(const char *linkto, const char *fullname); +int do_symlink(const char *contents, const char *fullname); +// Effect: If the symlink exists, return non-zero. + +void do_unlink(const char *fullname); +void do_rmdir(const char *fullname); +char *pathcat(const char *a, const char *b); + +// The main() compilation unit must provide this. +void help(int exitcode); +size_t parse_number_or_help(const char *numstring); + + +//Forward declaration of struct FTW as giving compilation error. +struct FTW; +int delete_subtree(const char *path, const struct stat *st, int typeflag, struct FTW *ftw); +void delete_dir_recursively(const char *path); + +/************************************************************************/ + +void +maybe_error(int r, const char *fun, const char *fname, const char *srcfile, int line); + +#define MAYBE_ERROR(fun, fname, args...) maybe_error(fun(fname, ##args), #fun, fname, __FILE__, __LINE__) + + +void +maybe_error2(int r, const char *fun, const char *fname, const char *fname2, const char *srcfile, int line); + +#define MAYBE_ERROR2(fun, fname1, fname2, args...) maybe_error2(fun(fname1, fname2, ##args), #fun, fname1, fname2, __FILE__, __LINE__) + +#endif diff --git a/software/fss-parallel-tools/sched.c b/software/fss-parallel-tools/sched.c new file mode 100644 index 0000000..4a74351 --- /dev/null +++ b/software/fss-parallel-tools/sched.c @@ -0,0 +1,241 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "sched.h" +#include "malloc.h" +#include +#include +#include +#include +#include "common/spew.h" + +enum state { RUNNING, SYNCHING, READY }; + +struct frame { + enum state state; + + // If READY we are in a list of all the ready and the continuation + struct frame *next, *prev; + continuation_t cont; + + // If SYNCING, we have a sync count + size_t sync_count; + + // If RUNNING there is no additional information. + + struct frame *parent; + inlet_t parent_return_inlet; + + void *app_frame; +}; + +static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t cond = PTHREAD_COND_INITIALIZER; + +static struct frame *ready; +static int all_done; + +static void push_ready_frame(struct frame *f) { + f->next = ready; + f->prev = NULL; + if (ready) ready->prev = f; + ready = f; + pthread_cond_signal(&cond); +} + +static struct frame *pop_ready_frame(void) { + while (!all_done && !ready) { + pthread_cond_wait(&cond, &mutex); + } + if (ready) { + assert(!all_done); + struct frame *result = ready; + ready->state = RUNNING; + if (result->next) { + result->next->prev = NULL; + } + ready = result->next; + return result; + } else { + assert(all_done); + return NULL; + } +} + + +struct frame * sched_spawn(struct frame *parent, + continuation_t parent_cont, + void *sub_app_frame, + inlet_t return_inlet) +{ + pthread_mutex_lock(&mutex); + struct frame *MALLOC(result); + spew(SPEW_DEBUG, "frame==%p", result); + *result = (struct frame){.state = RUNNING, + .next = NULL, + .prev = NULL, + .cont = NULL, + .sync_count = 0, + .parent = parent, + .parent_return_inlet = return_inlet, + .app_frame = sub_app_frame}; + parent->state = READY; + + push_ready_frame(parent); + + parent->cont = parent_cont; + + parent->sync_count++; + + pthread_mutex_unlock(&mutex); + return result; +} + +struct frame *sched_sync(struct frame *node, + continuation_t cont) { + pthread_mutex_lock(&mutex); + assert(node->state == RUNNING); + node->cont = cont; + struct frame *result; + if (node->sync_count == 0) { + result = node; + } else { + node->state = SYNCHING; + result = pop_ready_frame(); + } + pthread_mutex_unlock(&mutex); + return result; +} + +struct frame *sched_return(struct frame *node, size_t resultint) { + pthread_mutex_lock(&mutex); + struct frame *parent = node->parent; + assert(parent); + node->parent_return_inlet(parent->app_frame, resultint); + assert(parent->sync_count); + parent->sync_count--; + + struct frame *result; + switch(parent->state) { + case READY: /* nothing to do */ + case RUNNING: + result = pop_ready_frame(); + break; + case SYNCHING: + if (parent->sync_count == 0) { + parent->state = RUNNING; + result = parent; + } else { + result = pop_ready_frame(); + } + break; + default: abort(); + } + free(node); + pthread_mutex_unlock(&mutex); + return result; +} + +static void* worker(void*ignore __attribute__((unused))) { + pthread_mutex_lock(&mutex); + struct frame *f = pop_ready_frame(); + pthread_mutex_unlock(&mutex); + while (f) { + spew(SPEW_DEBUG, "f=%p cont=%p", f, f->cont); + assert(f->state == RUNNING); + f = f->cont(f, f->app_frame); + } + pthread_mutex_lock(&mutex); + assert(all_done); + pthread_mutex_unlock(&mutex); + return NULL; +} + +static void run_workers(size_t n_workers) { + pthread_t *MALLOC_N(workers, n_workers); + for (size_t i = 0; i < n_workers; i++) { + int r = pthread_create(&workers[i], NULL, worker, NULL); + assert(r == 0); + } + spew(SPEW_DEBUG, "starting workers"); + for (size_t i = 0; i < n_workers; i++) { + void *nullv = NULL; + int r = pthread_join(workers[i], nullv); + assert(r==0); + assert(nullv == NULL); + } + free(workers); +} + +magic_t sched_app_frame_magic = "app_frame_magic"; +struct sched_app_frame { + magic_t *magic; + continuation_t fun; + void *fun_app_frame; + size_t have_answer; + size_t answer; +}; +static void inlet_to_saf (void *safv, size_t r) { + struct sched_app_frame *saf = safv; + assert(saf->magic == &sched_app_frame_magic); + assert(!saf->have_answer); + saf->have_answer = 1; + saf->answer = r; +} +static struct frame *prun_wait(struct frame *frame, void *app_frame); +static struct frame *prun_finish(struct frame *frame, void *app_frame); +static struct frame *prun_start(struct frame *frame, void *app_frame) { + struct sched_app_frame *saf = app_frame; + spew(SPEW_DEBUG, "%p prun_start inlet_to_saf=%p", frame, inlet_to_saf); + assert(saf->magic == &sched_app_frame_magic); + spew(SPEW_DEBUG, "prun_wait=%p", prun_wait); + struct frame *fib_frame = sched_spawn(frame, prun_wait, saf->fun_app_frame, inlet_to_saf); + spew(SPEW_DEBUG, "fib_frame=%p, calling %p", fib_frame, saf->fun); + struct frame *next = saf->fun(fib_frame, saf->fun_app_frame); + spew(SPEW_DEBUG, "next=%p", next); + return next; +} +static struct frame *prun_wait(struct frame *frame, void *app_frame __attribute__((unused))) { + spew(SPEW_DEBUG, "prun_finish=%p", prun_finish); + return sched_sync(frame, prun_finish); +} +static struct frame *prun_finish(struct frame *frame __attribute__((unused)), void *app_frame __attribute__((unused))) { + assert(ready == NULL); + pthread_mutex_lock(&mutex); + all_done = 1; + pthread_cond_broadcast(&cond); + pthread_mutex_unlock(&mutex); + return NULL; +} +size_t prun(size_t n_workers, continuation_t fun, void *faf) { + struct sched_app_frame *MALLOC(saf); + *saf = (struct sched_app_frame){.magic = &sched_app_frame_magic, + .fun = fun, + .fun_app_frame = faf, + .have_answer = 0, + .answer = 0}; + struct frame *MALLOC(f); + spew(SPEW_DEBUG, "prun frame = %p", f); + *f = (struct frame){.state = READY, + .next = NULL, + .prev = NULL, + .cont = prun_start, + .sync_count = 0, + .parent = NULL, + .parent_return_inlet = NULL, + .app_frame = saf}; + ready = f; + all_done = 0; + spew(SPEW_DEBUG, "calling run_workers number of workers: %ld", n_workers); + run_workers(n_workers); + spew(SPEW_DEBUG, "back from run_workers workers"); + assert(saf->have_answer); + size_t r = saf->answer; + free(f); + free(saf); + return r; +} diff --git a/software/fss-parallel-tools/sched.h b/software/fss-parallel-tools/sched.h new file mode 100644 index 0000000..231a50c --- /dev/null +++ b/software/fss-parallel-tools/sched.h @@ -0,0 +1,31 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef SCHED_H +#define SCHED_H +#include + +struct frame; + +typedef struct frame *(*continuation_t)(struct frame *frame, void *app_frame); +typedef void (*inlet_t)(void *parent_app_frame, size_t result); + +struct frame * sched_spawn(struct frame *parent, + continuation_t parent_cont, + void *sub_app_frame, + inlet_t return_inlet); + +struct frame *sched_sync(struct frame *node, + continuation_t cont); + +struct frame *sched_return(struct frame *node, size_t result); + +size_t prun(size_t n_workers, continuation_t fun, void *arg); + +typedef const char *magic_t; + +#endif diff --git a/software/fss-parallel-tools/stringset.c b/software/fss-parallel-tools/stringset.c new file mode 100644 index 0000000..2a87fe5 --- /dev/null +++ b/software/fss-parallel-tools/stringset.c @@ -0,0 +1,86 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "stringset.h" +#include "hashfun.h" +#include "malloc.h" +#include +#include +struct stringset { + size_t n; + size_t limit; + char **strings; +}; +struct stringset *mk_stringset(void) { + struct stringset *MALLOC(r); + const size_t N = 2; + char **MALLOC_N(strings, N); + for (size_t i = 0; i < N; i++) { + strings[i] = NULL; + } + *r = (struct stringset){0, N, strings}; + return r; +} +struct stringset *stringset_destroy(struct stringset *stringset) { + for (size_t i = 0; i < stringset->limit; i++) { + if (stringset->strings[i]) { + FREE(stringset->strings[i]); + } + } + FREE(stringset->strings); + FREE(stringset); + return NULL; +} +int stringset_contains(const struct stringset *stringset, const char *string) { + size_t mask = stringset->limit -1; + assert((stringset->limit & mask) == 0); + size_t h = hash_string(string) & mask; + for (size_t count = 0; count < stringset->limit; count++) { + size_t hi = (h + count) & mask; + if (stringset->strings[hi] == NULL) return 0; + if (strcmp(stringset->strings[hi], string) == 0) return 1; + } + assert(0); +} +static void stringset_insert_internal(struct stringset *stringset, char *string) { + // Requires string is not in the list. String must have been + // malloc'd and now the stringset will own it. + size_t mask = stringset->limit -1; + assert((stringset->limit & mask) == 0); + size_t h = hash_string(string) & mask; + stringset->n++; + for (size_t count = 0; count < stringset->limit; count++) { + size_t hi = (h + count) & mask; + if (stringset->strings[hi] == NULL) { + stringset->strings[hi] = string; + return; + } + } + assert(0); +} + +void stringset_insert(struct stringset *stringset, const char *string) { + if (stringset_contains(stringset, string)) { + return; + } + if (stringset->n * 2 >= stringset->limit) { + struct stringset old = *stringset; + const size_t new_limit = 2 * stringset->limit; + char **MALLOC_N(newstrings, new_limit); + for (size_t i = 0; i < new_limit; i++) { + newstrings[i] = NULL; + } + *stringset = (struct stringset){0, new_limit, newstrings}; + for (size_t i = 0; i < old.limit; i++) { + if (old.strings[i]) { + stringset_insert_internal(stringset, old.strings[i]);; + } + } + FREE(old.strings); + } + stringset_insert_internal(stringset, strdup(string)); +} diff --git a/software/fss-parallel-tools/stringset.h b/software/fss-parallel-tools/stringset.h new file mode 100644 index 0000000..9f882f4 --- /dev/null +++ b/software/fss-parallel-tools/stringset.h @@ -0,0 +1,15 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef STRINGSET_H +#define STRINGSET_H +struct stringset; +struct stringset *mk_stringset(void); +struct stringset *stringset_destroy(struct stringset *stringset); +int stringset_contains(const struct stringset *stringset, const char *string); +void stringset_insert(struct stringset *stringset, const char *string); +#endif diff --git a/software/fss-parallel-tools/test_fib.c b/software/fss-parallel-tools/test_fib.c new file mode 100644 index 0000000..ac5dd1d --- /dev/null +++ b/software/fss-parallel-tools/test_fib.c @@ -0,0 +1,93 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "sched.h" +#include "malloc.h" +#include +#include +#include "common/spew.h" + +// Fib example + +magic_t pfib_magic = "pfib magic"; +struct pfib_frame { + magic_t *magic; + size_t n; + size_t sum; +}; +static struct pfib_frame *make_pfib_frame(size_t n) { + struct pfib_frame *MALLOC(result); + *result = (struct pfib_frame){.magic = &pfib_magic, + .n = n, + .sum = 0}; + spew(SPEW_DEBUG, "%p n=%lu", result, result->n); + return result; +} +static struct pfib_frame *cast_pfib_frame(void *af) { + assert(af); + struct pfib_frame *pf = af; + assert(pf->magic == &pfib_magic); + return pf; +} +struct frame *pfib(struct frame *frame, void *app_frame); +static struct frame *pfib2(struct frame *frame, void *app_frame); +static struct frame *pfib3(struct frame *frame, void *app_frame); +static struct frame *pfib4(struct frame *frame, void *app_frame); +static void pfib_return_inlet(void *parent_app_frame, size_t result) { + struct pfib_frame *pf = cast_pfib_frame(parent_app_frame); + pf->sum += result; +} +struct frame *pfib(struct frame *frame, void *app_frame) { + struct pfib_frame *f = cast_pfib_frame(app_frame); + spew(SPEW_DEBUG, "fib(%lu)", f->n); + if (f->n < 2) { + //sleep((unsigned int)f->n); + size_t r = f->n; + free(f); + return sched_return(frame, r); + } else { + struct pfib_frame *subf = make_pfib_frame(f->n-1); + return pfib(sched_spawn(frame, pfib2, subf, pfib_return_inlet), subf); + } +} +static struct frame *pfib2(struct frame *frame, void *app_frame) { + struct pfib_frame *f = cast_pfib_frame(app_frame); + struct pfib_frame *subf = make_pfib_frame(f->n-2); + return pfib(sched_spawn(frame, pfib3, subf, pfib_return_inlet), subf); +} +static struct frame *pfib3(struct frame *frame, void *app_frame __attribute__((unused))) { + return sched_sync(frame, pfib4); +} +static struct frame *pfib4(struct frame *frame, void *app_frame) { + struct pfib_frame *f = cast_pfib_frame(app_frame); + size_t sum = f->sum; + free(f); + return sched_return(frame, sum); +} + +static size_t fib(size_t n) { + struct pfib_frame *pf = make_pfib_frame((size_t)n); + size_t fn = prun(4, pfib, pf); + return fn; +} + +int main(int argc, char *argv[]) { + if (argc == 1) { + assert(0 == fib(0)); + assert(1 == fib(1)); + assert(1 == fib(2)); + assert(2 == fib(3)); + assert(3 == fib(4)); + assert(5 == fib(5)); + assert(8 == fib(6)); + assert(6765 == fib(20)); + } else { + int n = atoi(argv[1]); + printf("fib(%u)=%lu\n", n, fib((size_t)n)); + } + return 0; +} diff --git a/software/fss-parallel-tools/todo.c b/software/fss-parallel-tools/todo.c new file mode 100644 index 0000000..c4e3553 --- /dev/null +++ b/software/fss-parallel-tools/todo.c @@ -0,0 +1,131 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "todo.h" +#include "malloc.h" +#include +#include + +struct todo_entry { + struct todo_item *item; + struct todo_entry *next; +}; + +struct todo_list{ + pthread_mutex_t mutex; + pthread_cond_t cond; + struct todo_entry *entries; + size_t n_undone; // how many items haven't finished. +}; + +struct todo_list * +mk_todo_list(void) { + struct todo_list *MALLOC(tl); + pthread_mutex_init(&tl->mutex, NULL); + pthread_cond_init(&tl->cond, NULL); + tl->entries = NULL; + tl->n_undone = 0; + return tl; +} + +struct todo_list * +todo_list_destroy(struct todo_list *tl) { + assert(tl->entries == NULL); + assert(tl->n_undone == 0); + pthread_mutex_destroy(&tl->mutex); + pthread_cond_destroy(&tl->cond); + FREE(tl); + return NULL; +} + + +void +todo_list_push(struct todo_list *tl, + struct todo_item *item) { + struct todo_entry *MALLOC(entry); + pthread_mutex_lock(&tl->mutex); + *entry = (struct todo_entry){item, tl->entries}; + tl->entries = entry; + tl->n_undone++; + pthread_cond_signal(&tl->cond); + pthread_mutex_unlock(&tl->mutex); +} + +static struct todo_item * +todo_list_pop(struct todo_list *tl) { + pthread_mutex_lock(&tl->mutex); + while (tl->n_undone > 0 && tl->entries == NULL) { + pthread_cond_wait(&tl->cond, &tl->mutex); + } + struct todo_item *result; + if (tl->entries) { + struct todo_entry *entry = tl->entries; + tl->entries = entry->next; + result = entry->item; + FREE(entry); + } else { + assert(tl->n_undone == 0); + result = NULL; + } + pthread_mutex_unlock(&tl->mutex); + return result; +} + +static void +todo_list_decrement_undone(struct todo_list *tl) { + pthread_mutex_lock(&tl->mutex); + tl->n_undone--; + if (tl->n_undone == 0) { + pthread_cond_broadcast(&tl->cond); + } + pthread_mutex_unlock(&tl->mutex); +} + +typedef const char *magic_t; +magic_t wpair_magic; + +struct wpair { + magic_t *magic; + size_t threadnum; + struct todo_list *tl; + void (*work_on_item_function)(struct todo_list *, struct todo_item *); +}; + +static void* todo_worker(void *arg) { + struct wpair *p = arg; + assert(p->magic == &wpair_magic); + while (1) { + struct todo_item *item = todo_list_pop(p->tl); + if (item == NULL) break; + p->work_on_item_function(p->tl, item); + todo_list_decrement_undone(p->tl); + } + return arg; +} + +void todo_list_run(struct todo_list *tl, + size_t n_threads, + struct todo_item *first_item, + void (*work_on_item_function)(struct todo_list *tl, + struct todo_item *)) { + todo_list_push(tl, first_item); + pthread_t *MALLOC_N(threads, n_threads); + struct wpair *MALLOC_N(wpairs, n_threads); + for (size_t i = 0; i < n_threads; i++) { + wpairs[i] = (struct wpair){&wpair_magic, i, tl, work_on_item_function}; + pthread_create(&threads[i], NULL, todo_worker, &wpairs[i]); + } + for (size_t i = 0; i < n_threads; i++) { + void *retval; + pthread_join(threads[i], &retval); + assert(retval == &wpairs[i]); + } + assert(tl->entries == NULL); + assert(tl->n_undone == 0); + FREE(threads); + FREE(wpairs); +} diff --git a/software/fss-parallel-tools/todo.h b/software/fss-parallel-tools/todo.h new file mode 100644 index 0000000..e3ed3e6 --- /dev/null +++ b/software/fss-parallel-tools/todo.h @@ -0,0 +1,39 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef TODO_H +#define TODO_H + +/* This is a scheduler for a simple to-do list. You create a todo_list and + * then you start it running with some workers with an initial todo item. The + * workers pop things from the todo list (and the work function can push more + * things on) until everything is done. + */ + +#include + +struct todo_item; +struct todo_list; + +struct todo_list * +mk_todo_list(void); + +struct todo_list * +todo_list_destroy(struct todo_list *todo_list); + +void +todo_list_push(struct todo_list *tl, + struct todo_item *item); + +void +todo_list_run(struct todo_list *todo_list, + size_t n_threads, + struct todo_item *first_item, + void (*work_on_item_function)(struct todo_list *, + struct todo_item *)); + +#endif diff --git a/software/fss-parallel-tools/vec.c b/software/fss-parallel-tools/vec.c new file mode 100644 index 0000000..6b9bf0a --- /dev/null +++ b/software/fss-parallel-tools/vec.c @@ -0,0 +1,82 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#define _GNU_SOURCE +#include "vec.h" +#include "malloc.h" +#include + +struct vec { + size_t n; + size_t limit; + struct vecitem **items; +}; + +struct vec *mk_vec(void) { + struct vec *MALLOC(v); + size_t init = 2; + struct vecitem **MALLOC_N(items, init); + *v = (struct vec){0, init, items}; + return v; +} +struct vec *vec_destroy(struct vec *vec) { + assert(vec->n == 0); + FREE(vec->items); + FREE(vec); + return NULL; +} + +size_t vec_size(const struct vec *vec) { + return vec->n; +} + +struct vecitem *vec_fetch(const struct vec *vec, size_t i) { + if (i < vec->n) return vec->items[i]; + else return NULL; +} + +void vec_push(struct vec *vec, struct vecitem *item) { + if (vec->n >= vec->limit) { + vec->limit *= 2; + REALLOC(vec->items, vec->limit); + } + assert(vec->n < vec->limit); + vec->items[vec->n++] = item; +} + +struct vecitem* vec_pop(struct vec *vec) { + if (vec->n) { + if (vec->limit > 4 + && vec->n * 4 < vec->limit) + { + vec->limit /= 2; + assert(vec->n < vec->limit); + REALLOC(vec->items, vec->limit); + } + return vec->items[--vec->n]; + } else { + return NULL; + } +} + +struct vecitem* vec_peek(struct vec *vec) { + if (vec->n) return vec->items[vec->n-1]; + else return NULL; +} + +static int vec_compar(const void *av, const void *bv, void *compar_f) { + struct vecitem * const *ap = av; + struct vecitem * const *bp = bv; + int (*compar)(const struct vecitem *, const struct vecitem *) = compar_f; + return compar(*ap, *bp); +} + +void vec_sort(struct vec *vec, + int (*compar)(const struct vecitem *, const struct vecitem *)) { + qsort_r(vec->items, vec->n, sizeof(vec->items[0]), + vec_compar, compar); +} diff --git a/software/fss-parallel-tools/vec.h b/software/fss-parallel-tools/vec.h new file mode 100644 index 0000000..33b8bdc --- /dev/null +++ b/software/fss-parallel-tools/vec.h @@ -0,0 +1,40 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef VEC_H +#define VEC_H + +#include + +struct vec; // Defined by vec.c +struct vecitem; // Defined by the caller + +struct vec *mk_vec(void); +struct vec *vec_destroy(struct vec *vec); + +size_t vec_size(const struct vec *vec); + +struct vecitem *vec_fetch(const struct vec *vec, size_t i); + +void vec_push(struct vec *vec, struct vecitem *item); +// Add item to the end of vec. + +struct vecitem* vec_pop(struct vec *vec); +// Effect: If the vec is empty, return NULL, else remove and return +// the last item in the vec. + +struct vecitem* vec_peek(struct vec *vec); +// Effect: If the vec is empty, return NULL, else return the last item +// in the vec (without removing it). + +void vec_sort(struct vec *vec, + int (*compar)(const struct vecitem *, const struct vecitem *)); +// Effect: Sort the vector using compar (which has the same semantics +// as the compar function in qsort, except for the type is struct +// vecitem, not void*. + +#endif diff --git a/software/fss-parallel-tools/vector-of-strings.c b/software/fss-parallel-tools/vector-of-strings.c new file mode 100644 index 0000000..6b3600a --- /dev/null +++ b/software/fss-parallel-tools/vector-of-strings.c @@ -0,0 +1,74 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#include "vector-of-strings.h" +#include "malloc.h" +#include +#include + +struct vector_of_strings { + size_t n; + size_t size; + char **strings; +}; + +struct vector_of_strings * +mk_vector_of_strings(void) { + struct vector_of_strings *MALLOC(v); + *v = (struct vector_of_strings){0, 2, NULL}; + MALLOC_N(v->strings, v->size); + return v; +} + +struct vector_of_strings * +vector_of_strings_destroy(struct vector_of_strings *v) { + for (size_t i = 0; i < v->n; i++) { + FREE(v->strings[i]); + } + FREE(v->strings); + FREE(v); + return NULL; +} + +char* +vector_of_strings_pop(struct vector_of_strings *v) { + if (v->n == 0) return NULL; + return v->strings[-- v->n]; +} + +void +vector_of_strings_push(struct vector_of_strings *v, const char *string) { + if (v->n >= v->size) { + assert(v->size); + v->size *= 2; + REALLOC(v->strings, v->size); + } + assert(v->n < v->size); + v->strings[v->n++] = strdup(string); +} + +size_t +vector_of_strings_size(const struct vector_of_strings *v) { + return v->n; +} + +int +vector_of_strings_any(const struct vector_of_strings *v, + int (*predicate)(const char *string, void *extra), + void *extra) { + for (size_t i = 0; i < v->n; i++) { + int r = predicate(v->strings[i], extra); + if (r) return r; + } + return 0; +} + +char const * +vector_of_strings_fetch(const struct vector_of_strings *v, size_t idx) { + assert(idx < vector_of_strings_size(v)); + return v->strings[idx]; +} diff --git a/software/fss-parallel-tools/vector-of-strings.h b/software/fss-parallel-tools/vector-of-strings.h new file mode 100644 index 0000000..eda5404 --- /dev/null +++ b/software/fss-parallel-tools/vector-of-strings.h @@ -0,0 +1,40 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +#ifndef VECTOR_OF_STRINGS_H +#define VECTOR_OF_STRINGS_H +#include + +struct vector_of_strings; + +struct vector_of_strings * +mk_vector_of_strings(void); + +struct vector_of_strings * +vector_of_strings_destroy(struct vector_of_strings *v); + +char* +vector_of_strings_pop(struct vector_of_strings *v); + +void +vector_of_strings_push(struct vector_of_strings *v, const char *string); + +size_t +vector_of_strings_size(const struct vector_of_strings *v); + +char const * +vector_of_strings_fetch(const struct vector_of_strings *v, size_t i); + +int +vector_of_strings_any(const struct vector_of_strings *v, + int (*predicate)(const char *string, void *extra), + void *extra); +// Effect: If any string s in v returns predicate(s, extra)!=0 return +// one of those nonzero values. If all return zero, return 0. May +// call predicate zero or more times on every item in v. + +#endif diff --git a/software/fss-parallel-tools/version.h b/software/fss-parallel-tools/version.h new file mode 100644 index 0000000..9f552fb --- /dev/null +++ b/software/fss-parallel-tools/version.h @@ -0,0 +1,69 @@ +/** + * Copyright © 2019, Oracle and/or its affiliates. All rights reserved. + * + * Licensed under the Universal Permissive License v 1.0 as shown at + * http://oss.oracle.com/licenses/upl + * + */ +extern const char gitrev[]; +// #define VERSION is in the makefile +static const char VERSION_HISTORY[] = + "Version 1.58 partar --exclude to work for sub-directories.\n" + "Version 1.57 Added a --include-from option to parcp and fix a segfault.\n" + "Version 1.56 Fixed the bug causing createtar operation to fail and fixed some other minor bugs.\n" + "Version 1.55 Added delete_dir_recursively function in putils.c to delete a non-empty directory\n" + "Version 1.54 Fixed cmdline parser to properly parse --exclude=PATTERN to return PATTERN instead of NULL\n" + "Version 1.53 Fixed the bug of overwrite of existing files. Truncate file before writing into existing file\n" + "Version 1.52 when utimensat fails also return a nonzero exit code at the end of the run.\n" + "Version 1.51 more diagnostic information when utimensat fails. Also don't crash.\n" + "Version 1.50 link() handles preexisting files.\n" + "Version 1.49 Fixed 64GB file size limit bug.\n" + "Version 1.48 Improve diagostics when opendir fails.\n" + "Version 1.47 Added %{vendor} and %{group} to RPM spec.\n" + "Version 1.46 Replaced support e-mail with official alias.\n" + "Version 1.45 Minor documentation fixes and cleanups.\n" + "Version 1.44 License and Copyright notice iin every source file.\n" + "Version 1.43 handle setuid and setgid in parcp.\n" + "Version 1.42 got rid of puntar in the the RPM spec.\n" + "Version 1.41 partar handles read-only file in destination directory. Also partar is valgrind-clean.\n" + "Version 1.40 get rid of puntar. Long live partar.\n" + "Version 1.39 handle read-only files in destination.\n" + "Version 1.38 remove apache license.\n" + "Version 1.37 parcp had more bad interaction between --delete and --exclude-from.\n" + "Version 1.36 parcp had a bad interaction between --delete and --exclude-from.\n" + "Version 1.35 parcp has --exclude-from.\n" + "Version 1.34 added license files to RPMs.\n" + "Version 0.33 accept mtime==0 when partar is creating a tarball.\n" + "Version 0.32 spelling error in parcp --help.\n" + "Version 0.31 parcp doesn't try to delete .snapshot.\n" + "Version 0.30 parcp now supports --delete.\n" + "Version 0.29 parcp now is similar to rsync -raxH. There's a command line argument to turn those features on but they are on by default anyway for now with no way to turn them off.\n" + "Version 0.28 Fixed gitrev insertion in RPM builds.\n" + "Version 0.27 Honor umask for implicitly created dirs even when /proc/self/status doesn't give umask. Also less work on fdleak when rlim_cur is large.\n" + "Version 0.26 honor umask when creating directories that are a prefix of a filename but not otherwise explicitly mentioned.\n" + "Version 0.25 partar improved error messages if the input is compressed without providing for decompression.\n" + "Version 0.24 parcp improves error handling and messages for argument pathnames.\n" + "Version 0.23 rename puntar binary to partar. Keep the puntar version around for now, so as to not break people's scripts. Eventually we'll get rid of puntar.\n" + "Version 0.22 don't complain chown or chmod fails (undo 0.21 and go further). Also don't build man pages if no help2man is present (but don't crash the build)\n" + "Version 0.21 better diagnostics when chown fails\n" + "Version 0.20 better permissions preservation (for directories) rpm package doesn't require pv.\n" + "Version 0.19 better permissions and mtime preservation.\n" + "Version 0.18 RPM name changed to fss-parallel-tools.\n" + "Version 0.17 --help and --version go to stdout now to make OL6.x happy.\n" + "Version 0.16 --exclude option to x option.\n" + "Version 0.15 improved error diagonstics.\n" + "Version 0.14 --exclude option to c option.\n" + "Version 0.13 unlink in parallel in the same directory (is this better than 0.12?).\n" + "Version 0.12 parrm fails gracefully in the face of permissions trouble.\n" + "Version 0.11 Improve make rules for manual pages.\n" + "Version 0.10 Include manual pages.\n" + "Version 0.9 Parallel tar (so now \"puntar cf foo.tar foo/\" works).\n" + "Version 0.8 Separate out parallel scheduler from the code.\n" + "Version 0.7 Add help for parrm and parcp. Add -P argument to parrm. Clean up the makefile.\n" + "Version 0.6 Include parrm and parcp.\n" + "Version 0.5 Handle FIFO's.\n" + "Version 0.4 rpm packaging, and make it compile on OEL6 and FC27.\n" + "Version 0.3 Don't use too much memory when extracting a large file. Provide --progress.\n" + "Version 0.2 Cleanup.\n" + "Version 0.1 Delivered (1h4m reported) 1b39e0d817390bde5a880e614ae7dc54965e5d23.\n" +;