Main

Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.

The Universal Permissive License (UPL), Version 1.0

Subject to the condition set forth below, permission is hereby granted to any

person obtaining a copy of this software, associated documentation and/or data

(collectively the "Software"), free of charge and under any and all copyright

rights in the Software, and any and all patent rights owned or freely licensable

by each licensor hereunder covering either (i) the unmodified Software as

contributed to or provided by such licensor, or (ii) the Larger Works (as

defined below), to deal in both

(a) the Software, and

(b) any piece of software and/or hardware listed in the lrgrwrks.txt file if one

is included with the Software (each a “Larger Work” to which the Software is

contributed by such licensors),

without restriction, including without limitation the rights to copy, create

derivative works of, display, perform, and distribute the Software and make,

use, sell, offer for sale, import, export, have made, and have sold the Software

and the Larger Work(s), and to sublicense the foregoing rights on either these

or other terms.

This license is subject to the following condition:

The above copyright notice and either this complete permission notice or at a

minimum a reference to the UPL must be included in all copies or substantial

portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR

IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS

FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR

COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER

IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN

CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

##

# Copyright © 2019, Oracle and/or its affiliates. All rights reserved.

#

# Licensed under the Universal Permissive License v 1.0 as shown at

# http://oss.oracle.com/licenses/upl.

#

# Update this (and the VERSION_HISTORY in version.h) when the version changes.

VERSION = 1.58

#SAN = -fsanitize=thread

#SAN = -fsanitize=address

#GCOV_FLAGS = -fprofile-arcs -ftest-coverage

CFLAGS=-Wall -Wextra -Wshadow -Wconversion -Wpointer-arith -Wmissing-prototypes -Wwrite-strings -Wswitch-enum -Werror -g -O1 -pthread -std=gnu99 $(SAN) $(GCOV_FLAGS)

CPPFLAGS=-DVERSION=\"$(VERSION)\"

#CPPFLAGS+=-DSPEW_ENABLED

# It turns out that we need -lrt (probably because of clock_gettime()

# or else we'll end up with a dependency on glibc 2.17, which often is

# too advanced (e.g., on Oracle Linux 6.9)

LDLIBS=-lrt

LDFLAGS=-pthread $(SAN) $(GCOV_FLAGS)

VALGRIND = valgrind --leak-check=full --error-exitcode=3 -q

PARTAR = partar

default: $(PARTAR) parcp parrm $(PARTAR).1.gz parcp.1.gz parrm.1.gz

# Don't delete intermediate files (specify .SECONDARY with no dependencies)

.SECONDARY:

rpms: ../fss-parallel-tools-$(VERSION).tar.gz fss-parallel-tools.spec

mkdir -p ~/rpmbuild/SOURCES

mkdir -p ~/rpmbuild/SPECS

cp ../fss-parallel-tools-$(VERSION).tar.gz ~/rpmbuild/SOURCES

cp fss-parallel-tools.spec ~/rpmbuild/SPECS

cd ~/rpmbuild/SPECS;rpmbuild -ba fss-parallel-tools.spec

fss-parallel-tools.spec: fss-parallel-tools.spec.in Makefile

sed s/VERSION_DEFINED_IN_MAKEFILE/$(VERSION)/ < $< > $@

mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))

current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))

SOURCES = gitrev.c $(wildcard *.c *.h common/*.c common/*.h) Makefile LICENSE

../fss-parallel-tools-$(VERSION).tar.gz: $(SOURCES)

tar czf $@ --transform 's,^,fss-parallel-tools/,' $(SOURCES)

tarball: ../fss-parallel-tools-$(VERSION).tar.gz

todo.o: todo.h

dircache.o: dircache.h malloc.h

htable.o: htable.h malloc.h

fwg.o: malloc.h fwg.h hashfun.h

stringset.o: stringset.h hashfun.h malloc.h

####################

# COMMON

####################

# sources

common/assert.o: common/assert.h common/backtrace.h common/spew.h version.h

COMMON_O += common/assert.o

COMMON_O += common/backtrace.o

common/backtrace.o: common/spew.h common/simplest-defs.h common/sparse.h common/backtrace.h

COMMON_O += common/debug-tag.o

common/debug-tag.o: common/debug-tag.h common/simplest-defs.h common/sparse.h

COMMON_O += common/error.o

common/error.o: common/error.h common/simplest-defs.h common/sparse.h

COMMON_O += common/spew.o

common/spew.o: common/spew.h common/backtrace.h common/simplest-defs.h common/tsan.h version.h common/debug-tag.h common/sparse.h common/error.h

fdleak.o: fdleak.h

putils.o: putils.h common/spew.h

parcp.o: putils.h malloc.h version.h sched.h vector-of-strings.h common/spew.h

$(PARTAR).o: putils.h malloc.h version.h sched.h vector-of-strings.h inodemap.h stringset.h vec.h fdleak.h fwg.h common/spew.h

parrm.o: putils.h malloc.h version.h sched.h vector-of-strings.h common/spew.h

vector-of-strings.o: vector-of-strings.h malloc.h

$(PARTAR): $(PARTAR).o putils.o sched.o vector-of-strings.o gitrev.o fwg.o stringset.o vec.o fdleak.o inodemap.o $(COMMON_O)

inodemap.o: inodemap.h malloc.h

parcp: parcp.o putils.o sched.o vector-of-strings.o gitrev.o inodemap.o todo.o $(COMMON_O)

parrm: parrm.o putils.o gitrev.o sched.o vector-of-strings.o $(COMMON_O)

.DELETE_ON_ERROR:

.SUFFIX: .1.gz

%.1.gz: %

PATH=$$PATH:. help2man -N --name="parallel implementation of $(subst par,,$<)" $< | gzip - > $@

gitrev.o: gitrev.c

# Don't create a new gitrev.c file if nothing changed. We don't want

# to rerun all the tests just because I modified the Makefile.

gitrev.c: $(filter-out gitrev.c,$(wildcard *.c *.h common/*.c common/*.h Makefile))

echo \#include \"version.h\" > gitrev_new.c

echo const char gitrev[] = \"$$(git diff-index --quiet HEAD -- || echo "++")$$(git rev-parse HEAD)\"\; >> gitrev_new.c

if ! diff gitrev_new.c gitrev.c >> /dev/null ; then mv gitrev_new.c gitrev.c; fi

setup: $(PARTAR)

rm -rf tar-1.30

# Tests

.PHONY: check

TESTS = fifo tar-1.30 linux-4.15 longfilename3

TESTS_T = $(patsubst %, %.test, $(TESTS))

check: parrm_eaccess_test.test

check: parrm_eaccess_test2.test

check: parrm_eaccess_test3.test

check: parrm_eaccess_test4.test

check: excld.test

check: excld-equal.test

check: error.test

check: linklink.test

check: permissions.test

check: $(TESTS_T) test_fib.run

check: decompression-errors.test

check: parcp0.test parcp1.test parcp2.test parcp3.test parcp4.test parcp5.test parcp6.test parcp7.test

.SUFFIX: .test

%.test: %.partar_test %.parcp_test %.parrm_test %.tar_test

touch $@

.SUFFIX: .partar_test

%.partar_test: %.partar1_diff %.partar2_diff %.partar3_diff %.partar4_diff partar1.test

touch $@

test_fib.run: test_fib

./test_fib

touch $@

$(info VAR="$(COMMON_O)")

test_fib: test_fib.o sched.o $(COMMON_O) gitrev.o

test_fib.o: malloc.h sched.h common/spew.h

sched.o: malloc.h sched.h common/spew.h

#

# These tests run partar in various ways and compare the output to what happened with tar.

# partar_diff1_test runs "partar pxf %.tar"

# partar_diff2_test runs "partar pxf - < %.tar"

# partar_diff3_test runs "partar pxzf %.tar.gz"

# partar_diff4_test runs "partar pxzf - < %.tar.gz"

# partar_test has things that are fifos in it, and we want to ignore

# diff error messages about those. That's what the grep is about in

# the diff line.

output-with-tar output-with-partar1 output-with-partar2 output-with-partar3 output-with-partar4 uncompressed-tars gzipped-tars:

mkdir -p $@

uncompressed-tars/%.tar: test-tars/%.tar.xz | uncompressed-tars

xzcat $< > $@

gzipped-tars/%.tar.gz: uncompressed-tars/%.tar | gzipped-tars

gzip < $< > $@

%.tar_done: uncompressed-tars/%.tar | output-with-tar

rm -rf $(patsubst uncompressed-tars/%.tar,output-with-tar/%, $<)

tar xf $< -C output-with-tar

touch $@

# partar1 tests makes sure that partar works with a named file

%.partar1_done: uncompressed-tars/%.tar $(PARTAR) | output-with-partar1

rm -rf $(patsubst uncompressed-tars/%.tar,output-with-partar1/%, $<)

$(VALGRIND) ./$(PARTAR) xf $< -C output-with-partar1

touch $@

%.partar1_diff: %.tar_done %.partar1_done

diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar1/%,$<) | (! grep -v "is a fifo.*is a fifo")

touch $@

# partar2 tests make sure that partar works from stdin

%.partar2_done: uncompressed-tars/%.tar $(PARTAR) | output-with-partar2

rm -rf $(patsubst uncompressed-tars/%.tar,output-with-partar2/%, $<)

$(VALGRIND) ./$(PARTAR) xf - -C output-with-partar2 < $<

touch $@

%.partar2_diff: %.tar_done %.partar2_done

diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar2/%,$<) | (! grep -v "is a fifo.*is a fifo")

touch $@

# partar3 tests make sure that partar works with a gzipped named file.

.SUFFIX: .partar3_done

%.partar3_done: gzipped-tars/%.tar.gz $(PARTAR) | output-with-partar3

rm -rf $(patsubst gzipped-tars/%.tar.gz,output-with-partar3/%, $<)

$(VALGRIND) ./$(PARTAR) xzf $< -C output-with-partar3

touch $@

.SUFFIX: .partar3_diff

%.partar3_diff: %.tar_done %.partar3_done

diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar3/%,$<) | (! grep -v "is a fifo.*is a fifo")

touch $@

# partar4 tests make sure that partar works with a gzipped stdin

%.partar4_done: gzipped-tars/%.tar.gz $(PARTAR) | output-with-partar4

rm -rf $(patsubst gzipped-tars/%.tar.gz,output-with-partar4/%, $<)

$(VALGRIND) ./$(PARTAR) xzf - -C output-with-partar4 < $<

touch $@

%.partar4_diff: %.tar_done %.partar4_done

diff --no-dereference -r $(patsubst %.tar_done,output-with-tar/%,$<) $(patsubst %.tar_done,output-with-partar4/%,$<) | (! grep -v "is a fifo.*is a fifo")

touch $@

# Test untarring onto a read-only file

partar1.test: $(PARTAR) | partar1.test.dir

rm -rf $@.dir/src $@.dir/dst

mkdir -p $@.dir/src $@.dir/dst

touch $@.dir/dst/readonly

chmod guo-w $@.dir/dst/readonly

echo contents > $@.dir/src/readonly

chmod guo-w $@.dir/src/readonly

set -e; set -o pipefail; $(VALGRIND) ./$(PARTAR) c -C $@.dir/src readonly|$(VALGRIND) ./$(PARTAR) x -C $@.dir/dst

diff -r $@.dir/{src,dst}

touch $@

output-with-parcp output-to-rm output-of-partarred:

mkdir -p $@

.SUFFIX: .parcp_test

# Turn off valgrind for the big linux parcp

linux-4.15.parcp_test: VALGRIND=

%.parcp_test: parcp %.tar_done | output-with-parcp

rm -rf $(patsubst %.parcp_test,output-with-parcp/%, $@)

$(VALGRIND) ./parcp -P 3 $(patsubst %.parcp_test,output-with-tar/%, $@) $(patsubst %.parcp_test,output-with-parcp/%, $@)

diff --no-dereference -r $(patsubst %.parcp_test,output-with-tar/%, $@) $(patsubst %.parcp_test,output-with-parcp/%, $@) | (! grep -v "is a fifo.*is a fifo")

touch $@

.SUFFIX: .parrm_test

%.parrm_test: uncompressed-tars/%.tar parrm $(PARTAR) | output-to-rm

$(VALGRIND) ./$(PARTAR) xf - -C output-to-rm < $<

test -d $(patsubst %.parrm_test,output-to-rm/%,$@)

$(VALGRIND) ./parrm -P 3 $(patsubst %.parrm_test,output-to-rm/%,$@)

test ! -d $(patsubst %.parrm_test,output-to-rm/%,$@)

touch $@

# This test only works on a few cases, since the order of the files isn't unique

.SUFFIX: .tardiff_test

%.tardiff_test: %.tar_out %.partar_out

diff $^

touch $@

.SUFFIX: .tar_test

%.tar_test: %.partar_out %.tar_done | output-of-partarred

rm -rf $(patsubst %.partar_out,output-of-partarred/%,$<)

$(VALGRIND) ./$(PARTAR) xf $< -C output-of-partarred

diff --no-dereference -r $(patsubst %.tar_test,output-with-tar/%,$@) $(patsubst %.tar_test,output-of-partarred/%,$@) | (! grep -v "is a fifo.*is a fifo")

$(EXTRA_TEST)

# These two tests check to see what happens if parrm runs into a

# directory that it hasn't got permission to access. The whole

# directory, or a subdirectory, respectively.

ED = parrm_eaccess_test_dir

parrm_eaccess_test.test: parrm

-chmod u+rwx $(ED) > /dev/null 2>&1 ;true

rm -rf $(ED)

mkdir $(ED)

touch $(ED)/foo

chmod ugo-rwx $(ED)

rm -rf $(ED) > parrm_eaccess_test.out 2>&1 ; test $$? = 1

fgrep Permission parrm_eaccess_test.out > /dev/null

./parrm $(ED) > parrm_eaccess_test.out 2>&1 ; test $$? = 1

fgrep Permission parrm_eaccess_test.out > /dev/null

touch $@

ED2 = parrm_eaccess_test2_dir

parrm_eaccess_test2.test: parrm

chmod -R u+rwx $(ED2) > /dev/null 2>&1 ;true

rm -rf $(ED2)

mkdir -p $(ED2)/$(ED2)

touch $(ED2)/$(ED2)/foo

touch $(ED2)/foo

chmod ugo-rwx $(ED2)/$(ED2)

./parrm $(ED2) > $(ED2).out 2>&1 ; test $$? = 1

chmod -R u+rwx $(ED2) > /dev/null 2>&1 ;true

test ! -f $(ED2)/foo

test -f $(ED2)/$(ED2)/foo

touch $@

# And these are the same tests where we can access the dir but we cannot unlink it.

ED3 = parrm_eaccess_test3_dir

parrm_eaccess_test3.test: parrm

-chmod u+rwx $(ED3) > /dev/null 2>&1 ;true

rm -rf $(ED3)

mkdir $(ED3)

touch $(ED3)/foo

chmod ugo-w $(ED3)

ls -la $(ED3)

rm -rf $(ED3) > $(ED3).rm.out 2>&1 ; test $$? = 1

fgrep Permission $(ED3).rm.out > /dev/null

./parrm $(ED3) > $(ED3).parrm.out 2>&1 ; test $$? = 1

fgrep Permission $(ED3).parrm.out > /dev/null

touch $@

ED4 = parrm_eaccess_test4_dir

parrm_eaccess_test4.test: parrm

chmod -R u+rwx $(ED4) > /dev/null 2>&1 ;true

rm -rf $(ED4)

mkdir -p $(ED4)/$(ED4)

touch $(ED4)/$(ED4)/foo

touch $(ED4)/foo

chmod ugo-rx $(ED4)/$(ED4)

./parrm $(ED4) > $(ED4).out 2>&1 ; test $$? = 1

chmod -R u+rwx $(ED4) > /dev/null 2>&1 ;true

test ! -f $(ED4)/foo

test -f $(ED4)/$(ED4)/foo

touch $@

# Some tarballs have extra requirements that are hard to capture with diff. We'd really like to verify that the topology of two directories is the same, but I don't know how to get diff to really do it. So we'll have an ad hoc test that these two files really are the same.

longfilename3.tar_test: EXTRA_TEST = test output-of-partarred/longfilename3/aaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbccccccccccccccccdddddddddddddddeeeeeeeeeeeeeeffffffffffffffg -ef output-of-partarred/longfilename3/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb

.SUFFIX: .tar_out .partar_out

%.tar_out: %.tar_done

tar cf $@ $(patsubst %.tar_done,output-with-tar/%,$<)

%.partar_out: %.tar_done $(PARTAR)

(cd output-with-tar;../$(PARTAR) cf - $(patsubst %.tar_done,%,$<)) > $@

error.test: $(PARTAR)

! ./$(PARTAR) xf - < /dev/null 2> error.test.out

fgrep -q ": This does not look like a tar archive" error.test.out

fgrep -q ": Exiting with failure status due to previous errors" error.test.out

! ./$(PARTAR) xzf - < /dev/null 2> error.test.out

fgrep -q "gzip: stdin: unexpected end of file" error.test.out

fgrep -q ": Child returned status 1" error.test.out

fgrep -q ": Exiting with failure status due to previous errors" error.test.out

! ./$(PARTAR) cf error.tar output-with-tar/fifo no-such-file output-with-tar/fifo 2> error.test.out

fgrep -q ": no-such-file: Cannot stat: No such file or directory" error.test.out

fgrep -q ": Exiting with failure status due to previous errors" error.test.out

touch $@

# Test excluding files both in "c" mode and in "x" mode.

excld.test: tar-1.30.tar_done $(PARTAR)

rm -rf excld.dir.tar excld.dir.partar excld.odir.tar excld.odir.partar

mkdir excld.dir.tar excld.dir.partar excld.odir.tar excld.odir.partar

tar cf - -C output-with-tar/ --exclude d\?c tar-1.30 | tar xf - -C excld.dir.tar

$(VALGRIND) ./$(PARTAR) cf - -C output-with-tar/ --exclude d\?c tar-1.30 | tar xf - -C excld.dir.partar

diff -r excld.dir.tar excld.dir.partar

tar c -C output-with-tar/ tar-1.30 | tar x -C excld.odir.tar --exclude d\?c

diff -r excld.dir.tar excld.odir.tar

tar c -C output-with-tar/ tar-1.30 | $(VALGRIND) ./$(PARTAR) x -C excld.odir.partar --exclude d\?c

diff -r excld.dir.tar excld.odir.partar

touch $@

# Test excluding files both in "c" mode and in "x" mode same as above but uses --exclude=PATTERN.

excld-equal.test: tar-1.30.tar_done $(PARTAR)

rm -rf excld-equal.dir.tar excld-equal.dir.partar excld-equal.odir.tar excld-equal.odir.partar

mkdir excld-equal.dir.tar excld-equal.dir.partar excld-equal.odir.tar excld-equal.odir.partar

tar cf - -C output-with-tar/ --exclude=d\?c tar-1.30 | tar xf - -C excld-equal.dir.tar

$(VALGRIND) ./$(PARTAR) cf - -C output-with-tar/ --exclude=d\?c tar-1.30 | tar xf - -C excld-equal.dir.partar

diff -r excld-equal.dir.tar excld-equal.dir.partar

tar c -C output-with-tar/ tar-1.30 | tar x -C excld-equal.odir.tar --exclude d\?c

diff -r excld-equal.dir.tar excld-equal.odir.tar

tar c -C output-with-tar/ tar-1.30 | $(VALGRIND) ./$(PARTAR) x -C excld-equal.odir.partar --exclude=d\?c

diff -r excld-equal.dir.tar excld-equal.odir.partar

touch $@

# Test the same file appearing twice on the command line. tar makes a hardlink from the file to itself.

linklink.test: $(PARTAR)

echo a > linklink.txt

tar cf linklink.tar linklink.txt linklink.txt

$(VALGRIND) ./$(PARTAR) cf linklink.partar linklink.txt linklink.txt

# Don't run the diff unless we have a recent version of tar

if (tar --version|head -1|fgrep 1.3 > /dev/null); then diff linklink.tar linklink.partar; fi

mkdir -p linklink.tdir

tar xf linklink.tar -C linklink.tdir

mkdir -p linklink.pdir

tar xf linklink.partar -C linklink.pdir

diff -r linklink.tdir linklink.pdir

touch $@

# This isn't part of the standard tests, since it requires sudo to run properly.

permissions.sudo.test: $(PARTAR)

sudo rm -rf permissions.sudopartar.testdir permissions.sudountar.testdir

mkdir -p permissions.sudopartar.testdir

mkdir -p permissions.sudountar.testdir

sudo tar pxf test-tars/permissions.tar -C permissions.sudountar.testdir

sudo ./$(PARTAR) pxf test-tars/permissions.tar -C permissions.sudopartar.testdir

# for some reason tar doesn't set the mtime on partar/src/bin in those directories, so let's ignore bin.

(cd permissions.sudopartar.testdir; ls -lR)|sort | egrep -v "bin$$" > permissions.sudopartar.out

(cd permissions.sudountar.testdir; ls -lR)|sort | egrep -v "bin$$" > permissions.sudountar.out

diff permissions.sudopartar.out permissions.sudountar.out

# permissions test for not sudo

permissions.test: $(PARTAR)

rm -rf permissions.in.dir permissions.tar.out.dir permissions.partar.out.dir permissions.tar.ls permissions.partar.ls

mkdir -p permissions.in.dir/s

mkdir -p permissions.in.dir/s/d1

mkdir -p permissions.in.dir/s/d2

ls -l permissions.in.dir/s

chmod 770 permissions.in.dir/s/d1

chmod 750 permissions.in.dir/s/d2

ls -l permissions.in.dir/s

mkdir -p permissions.tar.out.dir

tar cf - -C permissions.in.dir s | tar xf - -C permissions.tar.out.dir

ls -l permissions.tar.out.dir/s | sort > permissions.tar.ls

mkdir -p permissions.partar.out.dir

tar cf - -C permissions.in.dir s | $(VALGRIND) ./$(PARTAR) xf - -C permissions.partar.out.dir

ls -l permissions.partar.out.dir/s | sort > permissions.partar.ls

diff permissions.tar.ls permissions.partar.ls

touch $@

# Test for detecting .gz and .xz inputs.

decompression-errors.test: ./partar

! ./partar xf test-tars/fifo.tar.xz 2> decompression-errors.out

fgrep "Input does not appear to be a tar file." decompression-errors.out > /dev/null

fgrep "Input appears to be xz compressed" decompression-errors.out > /dev/null

!(xzcat test-tars/fifo.tar.xz |gzip|./partar xf - 2> decompression-errors.out)

fgrep "Input does not appear to be a tar file." decompression-errors.out > /dev/null

fgrep "appears to be gzipped" decompression-errors.out > /dev/null

touch $@

%.dir:

mkdir $@

# Some specific tests for parcp

parcp0.test: parcp tar-1.30.tar_done | parcp0.test.dir

rm -rf $@.dir/*

./parcp --restore -- output-with-tar/tar-1.30 $@.dir

touch $@

parcp1.test: parcp

./parcp --version |fgrep $(VERSION) > /dev/null

./parcp --help |fgrep OPTION > /dev/null

touch $@

# test that if we copy into an existing file that is longer, that it's properly trunated

parcp2.test: parcp | parcp2.test.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst/src

echo "hello" > $@.dir/src/hello

echo "hello there" > $@.dir/dst/src/hello

ls -i $@.dir/dst/src/hello > $@.dir/original.inode

./parcp -P 3 $@.dir/src $@.dir/dst

ls -i $@.dir/dst/src/hello > $@.dir/new.inode

diff $@.dir/src/hello $@.dir/dst/src/hello

diff $@.dir/new.inode $@.dir/original.inode

touch $@

parcp3.test: parcp | parcp3.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst

# source and dest are the same symlink, so expect no change

ln -s unchanged $@.dir/src/uc

ln -s unchanged $@.dir/dst/uc

# source and dest are slightly different symlinks, the same length.

ln -s changed0 $@.dir/src/ch

ln -s changed1 $@.dir/dst/ch

# source and dest are different symlinks of different length.

ln -s shortch $@.dir/src/chlen

ln -s longerch $@.dir/dst/chlen

# source is symlink, dest is not there

ln -s notthere $@.dir/src/nt

# source is a symlink, dest is a dir

ln -s dest-was-dir $@.dir/src/wasdir

mkdir $@.dir/dst/wasdir

# source is a symlink, dest is a regular file.

ln -s dest-was-reg $@.dir/src/wasreg

touch $@.dir/dst/wasreg

#ls -l $@.dir/dst

ls -i parcp3.test.dir/dst/uc > $@.dir/original.uc.symlink

# source and dest are fifos. Make sure the inode is reused

mknod $@.dir/src/src_fifo_dst_fifo p

mknod $@.dir/dst/src_fifo_dst_fifo p

ls -i $@.dir/dst/src_fifo_dst_fifo > $@.dir/src_fifo_dst_fifo.original

# source is fifo and dest is regular. Make sure dest becomes fifo

mknod $@.dir/src/src_fifo_dst_reg p

touch $@.dir/dst/src_fifo_dst_reg

# source is regular and dest is fifo. Make sure dest becomes reg

touch $@.dir/src/src_reg_dst_fifo

mknod $@.dir/dst/src_reg_dst_fifo p

# source is file, dest is dir

touch $@.dir/src/src_reg_dst_dir

mkdir $@.dir/dst/src_reg_dst_dir

# source is dir, dest is file

mkdir $@.dir/src/src_dir_dst_reg

touch $@.dir/dst/src_dir_dst_reg

# two files that are the same inode in src, nothing in dst

touch $@.dir/src/src_two_inodes_dst_null.1

ln $@.dir/src/src_two_inodes_dst_null.1 $@.dir/src/src_two_inodes_dst_null.2

# two files the same inode in src, a file in dst

touch $@.dir/src/src_two_inodes_dst_reg.1

ln $@.dir/src/src_two_inodes_dst_reg.1 $@.dir/src/src_two_inodes_dst_reg.2

touch $@.dir/dst/src_two_inodes_dst_reg.1

# two files the same inode in src, two dirs in dst

touch $@.dir/src/src_two_inodes_dst_two_dirs.1

ln $@.dir/src/src_two_inodes_dst_two_dirs.1 $@.dir/src/src_two_inodes_dst_two_dirs.2

mkdir $@.dir/dst/src_two_inodes_dst_two_dirs.1

mkdir $@.dir/dst/src_two_inodes_dst_two_dirs.2

# three files the same inode in src, three files in dst, two of which are the same inode.

touch $@.dir/src/src_3_inodes_dst_3_files.1

ln $@.dir/src/src_3_inodes_dst_3_files.1 $@.dir/src/src_3_inodes_dst_3_files.2

ln $@.dir/src/src_3_inodes_dst_3_files.1 $@.dir/src/src_3_inodes_dst_3_files.3

touch $@.dir/dst/src_3_inodes_dst_3_files.1

ln $@.dir/dst/src_3_inodes_dst_3_files.1 $@.dir/dst/src_3_inodes_dst_3_files.2

touch $@.dir/dst/src_3_inodes_dst_3_files.3

# do the copy

$(VALGRIND) ./parcp $@.dir/src $@.dir/dst

ls -l $@.dir/dst

test -h $@.dir/dst/ch

ls -l $@.dir/dst/ch | fgrep "ch -> changed0"

ls -l $@.dir/dst/nt | fgrep "nt -> notthere"

ls -l $@.dir/dst/wasdir | fgrep "wasdir -> dest-was-dir"

ls -l $@.dir/dst/chlen | fgrep "chlen -> shortch"

test -h $@.dir/dst/wasreg

ls -l $@.dir/dst/wasreg| fgrep "wasreg -> dest-was-reg"

# test follows symlinks so we must do htis more carefully

ls -i parcp3.test.dir/dst/uc > $@.dir/new.uc.symlink

diff $@.dir/original.uc.symlink $@.dir/new.uc.symlink

ls -i $@.dir/dst/src_fifo_dst_fifo > $@.dir/src_fifo_dst_fifo.new

diff $@.dir/src_fifo_dst_fifo.new $@.dir/src_fifo_dst_fifo.original

test -p $@.dir/dst/src_fifo_dst_reg

test -f $@.dir/dst/src_reg_dst_fifo

test -f $@.dir/dst/src_reg_dst_dir

test -d $@.dir/dst/src_dir_dst_reg

test parcp3.test.dir/dst/src_two_inodes_dst_null.1 -ef parcp3.test.dir/dst/src_two_inodes_dst_null.2

test parcp3.test.dir/dst/src_two_inodes_dst_reg.1 -ef parcp3.test.dir/dst/src_two_inodes_dst_reg.2

test parcp3.test.dir/dst/src_3_inodes_dst_3_files.1 -ef parcp3.test.dir/dst/src_3_inodes_dst_3_files.2

test parcp3.test.dir/dst/src_3_inodes_dst_3_files.1 -ef parcp3.test.dir/dst/src_3_inodes_dst_3_files.3

touch $@

# Check --delete

parcp4.test: parcp | parcp4.test.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst

mkdir -p $@.dir/dst/a/b/c/d/e

touch $@.dir/dst/a/b/c/d/e/f

mkdir -p $@.dir/dst/a/b/c/d/e2

touch $@.dir/dst/a/b/c/d/e2/f

mkdir -p $@.dir/src/x/y/z

mkdir -p $@.dir/src/a/w

touch $@.dir/src/a/w/foo

$(VALGRIND) ./parcp --delete $@.dir/src $@.dir/dst

diff -r $@.dir/src $@.dir/dst

touch $@

# Check --exclude-from

parcp5.test: parcp | parcp5.test.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst

mkdir $@.dir/src/wanted

mkdir $@.dir/src/unwanted

touch $@.dir/src/wanted/file1.keep

touch $@.dir/src/wanted/file2.nokeep

touch $@.dir/src/wanted/file3.nokeep

echo unwanted > $@.dir/excludelist

echo wanted/file2.nokeep >> $@.dir/excludelist

echo file3.nokeep >> $@.dir/excludelist

$(VALGRIND) ./parcp --exclude-from=$@.dir/excludelist $@.dir/src $@.dir/dst

test -d $@.dir/dst/wanted

test -f $@.dir/dst/wanted/file1.keep

ls $@.dir/dst/wanted|fgrep -v file2.nokeep > /dev/null

ls $@.dir/dst/wanted|fgrep -v file3.nokeep > /dev/null

test ! -d $@.dir/dst/unwanted

touch $@

# Check --exclude-from with --delete

parcp6.test: parcp Makefile | parcp6.test.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst

mkdir -p $@.dir/src/src_wanted

mkdir -p $@.dir/dst/dst_wanted

mkdir -p $@.dir/dst/dst_extra

mkdir -p $@.dir/dst/dst_extra2

mkdir -p $@.dir/dst/dst_kill

touch $@.dir/dst/dst_wanted/keepme

touch $@.dir/dst/dst_extra/dst_wanted_file

touch $@.dir/dst/dst_extra2/keep2

touch $@.dir/dst/dst_extra2/killme

touch $@.dir/dst/dst_kill/killme2

echo dst_wanted > $@.dir/excludelist

echo dst_wanted_file >> $@.dir/excludelist

echo dst_extra2/keep2 >> $@.dir/excludelist

$(VALGRIND) ./parcp --exclude-from=$@.dir/excludelist --delete $@.dir/src $@.dir/dst

test -f $@.dir/dst/dst_wanted/keepme

test -f $@.dir/dst/dst_extra/dst_wanted_file

test -f $@.dir/dst/dst_extra2/keep2

test ! -f $@.dir/dst/dst_extra2/killme

test ! -f $@.dir/dst/dst_kill/killme2

test ! -d $@.dir/dst/dst_kill

touch $@

# Check target contains a read-only file

# Three cases: 1) No such file in src

# 2) Exactly the same file in src

# 3) A different file in src

parcp7.test: parcp | parcp7.test.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst

touch $@.dir/dst/no-such-file-in-src

chmod ugo-w $@.dir/dst/no-such-file-in-src

echo same in src > $@.dir/src/same-file-in-src

chmod ugo-w $@.dir/src/same-file-in-src

cp -a $@.dir/src/same-file-in-src $@.dir/dst/same-file-in-src

echo diffrent in src > $@.dir/src/diff-file-in-src

chmod ugo-w $@.dir/src/diff-file-in-src

echo diffrent in dst dst > $@.dir/dst/diff-file-in-src

chmod ugo-w $@.dir/dst/diff-file-in-src

ls -l $@.dir/{src,dst}

$(VALGRIND) ./parcp --delete $@.dir/src $@.dir/dst

ls -l $@.dir/{src,dst}

test ! -f $@.dir/dst/no-such-file-in-src

test -f $@.dir/dst/diff-file-in-src

test -f $@.dir/dst/same-file-in-src

diff -r $@.dir/{src,dst}

touch $@

parcp8.test: parcp | parcp7.test.dir

rm -rf $@.dir/*

mkdir -p $@.dir/src $@.dir/dst

touch $@.dir/src/rws-file

chmod +sx $@.dir/src/rws-file

./parcp $@.dir/src $@.dir/dst

test -x parcp8.test.dir/dst/rws-file

test -u parcp8.test.dir/dst/rws-file

echo touch $@

.PHONY: clean

clean:

rm -rf parrm parcp $(PARTAR) parrm.1.gz parcp.1.gz $(PARTAR).1.gz *.o output-with-tar output-to-rm output-with-parcp *.tar_done *.parrm_test *.parcp_test *.partar_test

rm -rf gitrev.c

rm -rf output-with-partar{1,2,3,4} output-of-partarred

rm -rf uncompressed-tars gzipped-tars

rm -rf *.partar{1,2,3,4}_diff

rm -rf *.partar{1,2,3,4}_done

rm -rf excld.dir.*tar excld.test

rm -rf excld-equal.dir.*tar excld-equal.test

if test -n "$$(find . -maxdepth 1 -name 'parrm_eaccess_test*_dir' -print -quit)"; then chmod -R u+rwx parrm_eaccess_test*_dir; fi

rm -rf parrm_eaccess_test*{.out,_dir}

rm -rf error.test.out error.tar linklink.tar linklink.partar linklink.tdir linklink.pdir linklink.txt excld.odir.partar excld.odir.tar

rm -rf permissions.in.dir permissions.tar.out.dir permissions.partar.out.dir permissions.tar.ls permissions.partar.ls

rm -rf *.ls *.test *.tar_test

rm -rf *.partar_out *.tar_out test1.out *.out

rm -rf test_fin

rm -f *.gcda *.gcno common/*.gcda common/*.gcno

rm -rf *.dir/

rm -rf *.1.gz

rm -f $(COMMON_O)

rm -rf partar_log

.PHONY: realclean

realclean: clean

/**

* Copyright © 2019, Oracle and/or its affiliates. All rights reserved.

*

* Licensed under the Universal Permissive License v 1.0 as shown at

* http://oss.oracle.com/licenses/upl

*

*/

#include <assert.h>

#include <sys/stat.h>

#include <pthread.h>

#include <dirent.h>

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <unistd.h>

#include "putils.h"

#include "malloc.h"

/* parallel diff of a directory heirarchy that also checks ownership and attributes */

struct diff_task {

char *relative_path;

struct diff_task *next;

};

// Worker state

pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; ;

pthread_cond_t cond = PTHREAD_COND_INITIALIZER;

struct diff_task *tasks = NULL;

enum { n_workers = 32};

size_t n_working = 0; // When tasks==NULL and n_working==0, we are done

pthread_t workers[n_workers];

#define PMAX 4096

const char *src_dir;

const char *dst_dir;

int check_ownership = 1;

// From the qsort man page:

static int cmpstringp(const void *p1, const void *p2) {

// The actual arguments to this function are "pointers to pointers

// to char", but strcmp(3) arguments are "pointers to char", hence

// the following cast plus dereference.

return strcmp(* (char * const *) p1, * (char * const *) p2);

}

static void collect_and_sort_pathnames(const char *abs_path,

// output:

size_t *np,

char ***paths_p)

// Collect all the pathnames in a directory, except for . and .., and

// return them in sorted order.

{

size_t n = 0;

size_t limit = 2;

char **MALLOC_N(paths, limit);

DIR *dir = opendir(abs_path);

assert(dir);

struct dirent *dentry;

while ((dentry = readdir(dir))) {

if (strcmp(dentry->d_name, ".") == 0) continue;

if (strcmp(dentry->d_name, "..") == 0) continue;

if (n >= limit) {

limit *= 2;

REALLOC(paths, limit);

}

assert(n < limit);

paths[n++] = strdup(dentry->d_name);

}

closedir(dir);

qsort(paths, n, sizeof(char*), cmpstringp);

*np = n;

*paths_p = paths;

}

static void diff_r(const char *relative_path) {

char src_path[PMAX];

{

int n = snprintf(src_path, PMAX, "%s/%s", src_dir, relative_path);

assert(n < PMAX);

}

char dst_path[PMAX];

{

int n = snprintf(dst_path, PMAX, "%s/%s", dst_dir, relative_path);

assert(n < PMAX);

}

struct stat src_stat, dst_stat;

{

int src_r = lstat(src_path, &src_stat);

assert (src_r == 0);

}

{

int dst_r = lstat(dst_path, &dst_stat);

assert(dst_r == 0);

}

assert((src_stat.st_mode & S_IFMT) == (dst_stat.st_mode & S_IFMT));

{

mode_t mask = 0777;

if ((src_stat.st_mode & mask) != (dst_stat.st_mode & mask)) {

fprintf(stderr, "masks differ %s and %s\n", src_path, dst_path);

assert(0);

}

}

if (check_ownership) {

if (src_stat.st_uid != dst_stat.st_uid) {

fprintf(stderr, "uid doesn't match for %s and %s\n", src_path, dst_path);

}

assert(src_stat.st_uid == dst_stat.st_uid);

assert(src_stat.st_gid == dst_stat.st_gid);

}

if (S_ISREG(dst_stat.st_mode)) {

assert(src_stat.st_size == dst_stat.st_size);

FILE *src_f = fopen(src_path, "r");

FILE *dst_f = fopen(dst_path, "r");

if (src_f == NULL || dst_f == NULL) {

fprintf(stderr, "Failed to open one or more files\n");

return;

}

for (size_t i = 0 ; i < (size_t)(src_stat.st_size); i++) {

int src_c = getc(src_f); assert(src_c != EOF);

int dst_c = getc(dst_f); assert(dst_c != EOF);

assert(src_c == dst_c);

}

fclose(src_f);

fclose(dst_f);

} else if (S_ISDIR(src_stat.st_mode)) {

size_t sn, dn;

char **sps, **dps;

collect_and_sort_pathnames(src_path, &sn, &sps);

collect_and_sort_pathnames(dst_path, &dn, &dps);

assert(sn == dn);

for (size_t i = 0; i < sn; i++) {

assert(strcmp(sps[i], dps[i]) == 0);

struct diff_task *MALLOC(task);

char sub_rel_path[PMAX];

{

int n = snprintf(sub_rel_path, PMAX, "%s/%s", relative_path, sps[i]);

assert(n < PMAX);

}

char *srp = strdup(sub_rel_path);

pthread_mutex_lock(&mutex);

*task = (struct diff_task){.relative_path = srp,

.next = tasks};

tasks = task;

pthread_cond_signal(&cond);

pthread_mutex_unlock(&mutex);

}

} else if (S_ISLNK(src_stat.st_mode)) {

char slname[PMAX];

ssize_t slen = readlink(src_path, slname, PMAX);

assert(slen >= 0 && slen < PMAX);

char dlname[PMAX];

ssize_t dlen = readlink(dst_path, dlname, PMAX);

assert(dlen >= 0 && dlen < PMAX);

assert(slen == dlen);

slname[slen] = 0;

dlname[dlen] = 0;

if (0 != strcmp(slname, dlname)) {

fprintf(stderr, "%s -> %s is not the same as %s -> %s\n", src_path, slname, dst_path, dlname);

assert(0);

}

}

}

static void* run_worker(void *d) {

pthread_mutex_lock(&mutex);

while (1) {

// Lock is held here.

while (!tasks && n_working) {

pthread_cond_wait(&cond, &mutex);

}

// Either there is a task or n_working==0

if (tasks) {

struct diff_task *task = tasks;

tasks = task->next;

n_working++;

pthread_mutex_unlock(&mutex);

diff_r(task->relative_path);

free(task->relative_path);

free(task);

pthread_mutex_lock(&mutex);

n_working--;

if (tasks == NULL && n_working == 0) {

pthread_cond_broadcast(&cond);

}

} else if (n_working == 0) {

// No tasks and nothing to do.

pthread_mutex_unlock(&mutex);

return d;

}

}

}

static void run_workers(void) {

MALLOC(tasks);

*tasks = (struct diff_task){.relative_path = strdup(""),

.next = NULL};

n_working = 0;

for (size_t i = 0; i < n_workers; i++) {

pthread_create(&workers[i], NULL, run_worker, &workers[i]);

}

for (size_t i = 0; i < n_workers; i++) {

void *result;

pthread_join(workers[i], &result);

}

}

int main (int argc, const char *argv[]) {

assert(argc == 3);

src_dir = argv[1];

dst_dir = argv[2];

run_workers();

return 0;

}

#include "assert.h"

#include <inttypes.h> // for PRIx64

#include <stddef.h> // for size_t

#include <stdint.h> // for uint64_t

#include <stdlib.h> // for abort

#include <stdarg.h> // for va_end

#include "backtrace.h" // for spew_backtrace

#include "spew.h" // for log0, ::SPEW_FATAL, the_spew_log

#include "../version.h"

void assert_fail(const char *assertion,

const char *file, size_t line, const char *function)

{

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function,

"ASSERTION FAILURE: %s", assertion);

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function, "GITREV %s", gitrev);

spew_backtrace(SPEW_FATAL);

abort();

}

void assert_fail_int(const char *assertion, const char *str_var, int var,

const char *file, size_t line, const char *function)

{

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function, "ASSERTION FAILURE: %s, %s=%d",

assertion, str_var, var);

spew_backtrace(SPEW_FATAL);

abort();

}

void assert_fail_v(const char *assertion,

const char *file,

size_t line,

const char *function,

const char *fmt_string,

...) {

va_list args;

#ifndef __CHECKER__

// va_start() contains a cast to void * (from char const **, in

// this case), which runs afoul of sparse's check for qualifier

// drops.

//

va_start(args, fmt_string);

#endif // !__CHECKER__

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function,

"ASSERTION FAILURE: %s", assertion);

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function, "GITREV %s", gitrev);

logv(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function, fmt_string, args);

spew_backtrace(SPEW_FATAL);

abort();

va_end(args);

}

void check_eq_u(uintmax_t a, uintmax_t b,

const char *file, size_t line, const char *function,

const char *assertion)

{

if (a != b) {

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function,

"ASSERTION FAILURE: [%s]: 0x%"PRIxMAX" != 0x%"PRIxMAX,

assertion, a, b);

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function, "GITREV %s", gitrev);

spew_backtrace(SPEW_FATAL);

abort();

}

}

void check_eq_i(intmax_t a, intmax_t b,

const char *file, size_t line, const char *function,

const char *assertion)

{

if (a != b) {

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function,

"ASSERTION FAILURE [%s]: %"PRIiMAX" != %"PRIiMAX,

assertion, a, b);

log0(&the_spew_log, /*honor_spewage_suppression=*/0, /*maybe_roll=*/1,

/*is_stack_trace=*/0,

SPEW_FATAL, file, line, function, "GITREV %s", gitrev);

spew_backtrace(SPEW_FATAL);

abort();

}

}

#ifndef __COMMON_ASSERT_H__

#define __COMMON_ASSERT_H__

#include <stddef.h> // for size_t

#include <stdint.h> // for uint64_t

#include "simplest-defs.h" // for STRINGIFY

// Use CHECK for checking something that you really want checked.

//

// Use ASSERT for checking something that you want checked, but the

// functionality of the program won't be compromised if the predicate isn't

// checked, or even run. It should be correct to define ASSERT as "#define

// ASSERT(x) ((void)0)". This means that you should not put side effects into

// ASSERT predicates.

//

// Use ASSERT_WITH_SIDE_EFFECTS for checking something you want checked, but

// there may be side effects in the predicate, so we need need to evaluate the

// predicate even if NOASSERT is set.

//

// See wisdom.tex for more discussion.

#ifndef CHECK_ASSERT_SIDE_EFFECTS

#define CHECK_ASSERT_SIDE_EFFECTS 0

#endif

#ifdef NOASSERT

#define ASSERT(x) ((0) ? ((void)(x), (void)(0)) : (void)(0))

#define ASSERT0 ASSERT

#define ASSERTv(x, typeof_val, val) ((0) ? ((void)(x), (void)(val), (void)(0)) : (void)(0))

#define ASSERT_EQ_u(a, b) ((0) ? ((void)(a), (void)(b), (void)(0)) \

: (void)(0))

#define ASSERT_EQ_i(a, b) ((0) ? ((void)(a), (void)(b), (void)(0)) \

: (void)(0))

#define ASSERT_WITH_SIDE_EFFECTS(expr) ((void)expr)

#elif !CHECK_ASSERT_SIDE_EFFECTS // && !NOASSERT

#define ASSERT CHECK

#define ASSERT0 CHECK0

#define ASSERTv CHECKv

#define ASSERT_EQ_u CHECK_EQ_u

#define ASSERT_EQ_i CHECK_EQ_i

#define ASSERT_WITH_SIDE_EFFECTS ASSERT

#else // CHECK_ASSERT_SIDE_EFFECTS && !NOASSERT

// See https://stackoverflow.com/questions/10593492/catching-assert-with-side-effects

// for this trick for noticing that you have an ASSERT with side effects.

extern int not_needed_if_expr_has_no_side_effects;

#define ASSERT(expr) ((void)(not_needed_if_expr_has_no_side_effects || (expr)))

#define ASSERT0(expr) ASSERT((expr) == 0)

#define ASSERTv(x, typeof_val, val) ASSERT(x)

#define ASSERT_EQ_u(a, b) ASSERT((a) == (b))

#define ASSERT_EQ_i(a, b) ASSERT((a) == (b))

#define ASSERT_WITH_SIDE_EFFECTS(expr) ((void)(expr))

#endif // CHECK_ASSERT_SIDE_EFFECTS && !NOASSERT

#define CHECK(x) ((x) ? (void) (0) : assert_fail (STRINGIFY(x), __FILE__, __LINE__, __PRETTY_FUNCTION__))

#define CHECK0(x) CHECK((x) == 0)

/* Check and print variable of specified type. */

#define CHECKv(x, typeof_val, val) ((x) ? (void) (0) : assert_fail_ ## typeof_val \

(STRINGIFY(x), #val, val, __FILE__, __LINE__, __PRETTY_FUNCTION__))

#define CHECKvv(x, fmt_args...) ((x) ? (void)(0) : assert_fail_v(STRINGIFY(x), __FILE__, __LINE__, __PRETTY_FUNCTION__, fmt_args) )

#define UNREACHABLE() CHECK("UNREACHABLE" && 0)

void assert_fail(const char *assertion,

const char *file, size_t line, const char *function)

__attribute__ ((__noreturn__));

void assert_fail_int(const char *assertion, const char *str_var, int var,

const char *file, size_t line, const char *function)

__attribute__ ((__noreturn__));

void assert_fail_v(const char *assertion,

const char *file,

size_t line,

const char *function,

const char *fmt_string,

...)

__attribute__ ((__noreturn__))

__attribute__ ((format (printf, 5, 6)));

void

check_eq_u(uintmax_t a, uintmax_t b,

const char *file, size_t line, const char *function,

const char *assertion);

#define CHECK_EQ_u(a, b) ( \

check_eq_u((uintmax_t)(a), (uintmax_t)(b), \

__FILE__, __LINE__, __PRETTY_FUNCTION__, \

STRINGIFY(a) " == " STRINGIFY(b)))

void

check_eq_i(intmax_t a, intmax_t b,

const char *file, size_t line, const char *function,

const char *assertion);

#define CHECK_EQ_i(a, b) ( \

check_eq_i((intmax_t)(a), (intmax_t)(b), \

__FILE__, __LINE__, __PRETTY_FUNCTION__, \

STRINGIFY(a) " == " STRINGIFY(b)))

#endif

#define _GNU_SOURCE

#include "backtrace.h"

#include <execinfo.h> // for backtrace, backtrace_symbols

#include <fcntl.h> // for open, O_CLOEXEC, O_RDONLY

#include <pthread.h> // for pthread_mutex_lock, pthread_mute...

#include <spawn.h> // for posix_spawn_file_actions_t, posi...

#include <stddef.h> // for ptrdiff_t

#include <stdio.h> // for snprintf, sscanf

#include <stdlib.h> // for size_t, NULL, abort, free, strtoull

#include <string.h> // for memset, strrchr

#include <sys/wait.h> // for waitpid

#include <unistd.h> // for close, read, pipe2, ssize_t, rea...

#include "simplest-defs.h" // for NELEM, UNUSED

#include "sparse.h" // for __force

#include "spew.h" // for spew, SPEW_DEBUG, SPEW_INFO, SPE...

/* DO NOT USE ASSERT() IN THIS FILE, TO AVOID INFINITE RECURSION */

#ifdef ASSERT

#error CANNOT_USE_ASSERT_IN_THIS_FILE

#endif

// I'm using pthread_mutex_t rather than xthread_mutex_t because I cannot

// think of a better way to do this than with a static variable. And with a

// static variable, I want to use PTHREAD_MUTEX_INITIALIZER, which there

// doesn't seem to have a xthread_mutex analogue.

static pthread_mutex_t backtrace_mutex = PTHREAD_MUTEX_INITIALIZER;

// Backtracer wants to fork an "addr2line" process to be able to translate

// code offsets into human-readable strings. Back in the dark ages, we would

// do the fork whenever we wanted a backtrace. That turned out to be too

// expensive, since fork() would copy huge page table (covering many gigabytes

// of memory), and it was silly because the exec() would just throw it away.

//

// Our next solution used a constructor/destructor. At the beginning of the

// program we would start up the backtracer. At that point in time the page

// table was small, and we didn't really care if it was expensive since

// nothing was really happening yet. However we don't really like the

// constructor/destructor stuff, since it's fragile. (And Matteo doesn't like

// it because it's a desugared version of C++ :-)

//

// This solution uses posix_spawnp() to do a fork/exec operation.

// posix_spawnp() lets you do some housekeeping (with dup2 and close

// operations) between the fork and the spawn. Because the kernel gets all

// the information about the fork/dup/close/spawn sequence at once, it can

// avoid copying the page tables. The performance of this sequence is about

// 200us to 500us.

struct backtracer {

int broken; // true if something went wrong and the backtracer

// is failing. Fall back to not printing

// backtraces, and try to keep running.

pid_t child; // The addr2line process

int pipe_to_child; // The write end of a pipe that goes the

// addr2line's stdin.

int pipe_from_child; // The read end of a pipe that comes from

// addr2line's stdout.

size_t va_base; // The offset to be subtracted from addresses to

// to ASLR.

};

static int __attribute__((warn_unused_result)) // NEGATIVE_ON_ERROR

sinit(posix_spawn_file_actions_t *file_actions)

// Effect: Initialize the file_actions. If anything goes wrong return -1.

{

int r = posix_spawn_file_actions_init(file_actions);

if (r != 0) {

spew(SPEW_DEBUG, "Cannot spawn_file_actions_init: r=%d", r);

return -1;

} else {

return 0;

}

}

static int __attribute__((warn_unused_result)) // NEGATIVE_ON_ERROR

sdup2(posix_spawn_file_actions_t *file_actions, int from, int to)

// Effect: Record a dup2() operation to be done when posix_spawn runs. If

// anything goes wrong, spew a diagnostic and return -1;

{

int r = posix_spawn_file_actions_adddup2(file_actions, from, to);

if (r != 0) {

spew(SPEW_DEBUG, "Cannot spawn_fileactions_adddup2: r=%d", r);

return -1;

} else {

return 0;

}

}

static int __attribute__((warn_unused_result)) // NEGATIVE_ON_ERROR

sclose(posix_spawn_file_actions_t *file_actions, int fd)

// Effect: Record a close() operation to be cone when posix_spawn runs. If

// anything goes wrong, spew a diagnostic and return -1;

{

int r = posix_spawn_file_actions_addclose(file_actions, fd);

if (r != 0) {

spew(SPEW_DEBUG, "Cannot spawn_fileactions_addclose: r=%d", r);

return -1;

} else {

return 0;

}

}

static const struct backtracer errresult = {

.pipe_to_child=-1, .pipe_from_child=-1, .child=0, .broken=1, .va_base = 0};

static struct backtracer

mk_backtracer(void) {

// Only one backtracer may exist at a time (to prevent forkbombing if all

// threads start backtracing.) This mutex is unlocked when you destroy

// the backtracer.

pthread_mutex_lock(&backtrace_mutex);

// Plumbing

int tochildpipe[2];

int fromchildpipe[2];

{

int r = pipe2(tochildpipe, O_CLOEXEC);

if (r) {

spew(SPEW_ERROR, "pipe2 failed");

}

}

{

int r = pipe2(fromchildpipe, O_CLOEXEC);

if (r) {

spew(SPEW_ERROR, "pipe2 failed");

}

}

// Find the path to the current executable. It's a symlink in

// /proc/self/exe.

char exec_path[204];

// readlink doesn't null terminate. Fill it with zeros and tell it to

// only use sizeof(exec_path)-1 to make sure there is a null at the

// end.

memset(exec_path, 0, sizeof(exec_path));

ssize_t rc = readlink("/proc/self/exe", exec_path,

sizeof(exec_path)-1);

UNUSED(rc);

spew(SPEW_INFO, "exec path is %s", exec_path);

// Set up to do the plumbing after the spawn.

posix_spawn_file_actions_t file_actions;

if (0

|| sinit(&file_actions)

|| sdup2(&file_actions, tochildpipe[0], 0)

|| sdup2(&file_actions, fromchildpipe[1], 1)

|| sclose(&file_actions, tochildpipe[0])

|| sclose(&file_actions, tochildpipe[1])

|| sclose(&file_actions, fromchildpipe[0])

|| sclose(&file_actions, fromchildpipe[1]))

{

return errresult;

}

pid_t child;

{

// Hack around because posix_spawnp doesn't have const's in quite the right place.

char addr2line_s[] = "addr2line";

char _e_s[] = "-e";

char _f_s[] = "-f";

char * const argv[] = {addr2line_s, _e_s, exec_path, _f_s, NULL};

int r = posix_spawnp(&child, "addr2line",

&file_actions, NULL,

argv, environ);

if (r != 0) {

spew(SPEW_DEBUG, "Could not posix_spawnp r=%d", r);

return errresult;

}

}

{

int r = posix_spawn_file_actions_destroy(&file_actions);

if (r) spew(SPEW_FATAL, "Cannot destroy fail_actions r=%d", r);

}

// Plumbing back here the main program.

struct backtracer result = {.broken = 0,

.child = child,

.pipe_to_child = tochildpipe[1],

.pipe_from_child = fromchildpipe[0],

.va_base = 0};

close(tochildpipe[0]);

close(fromchildpipe[1]);

// Find the va_base

{

int fd = open("/proc/self/maps", O_RDONLY);

if (fd >= 0) {

char addr_buffer[256];

// HACK: we're going to assume that the first line in the

// file contains the mapping for our executable. Dynamic

// libraries will be mapped in different locations, but we

// don't really use any of them so our traceback should be

// mostly correct.

ssize_t bytes_read = read(fd, &addr_buffer, NELEM(addr_buffer)-1);

if (bytes_read > 0) {

addr_buffer[bytes_read] = 0;

int rval = sscanf(addr_buffer, "%zx-", &result.va_base);

if (rval != 1) {

result.va_base = 0;

}

}

}

close(fd);

}

spew(SPEW_INFO, "va_base is %zx", result.va_base);

if (result.va_base < 0x800000) {

// If ASLR is off, subtracting va_base makes offsets that addr2line

// doesn't understand. The best heuristic I've (jmp) found is that if

// va_base is "low", then ASLR is off. I haven't been able to figure

// out if there's a "low" address under which ASLR is disabled, but

// I've seen 4 MiB in many systems, so I'm going to claim any va_base

// under 8 MiB is in a non-ASLR system.

result.va_base = 0;

}

return result;

}

static void

backtracer_destroy(struct backtracer *bt) {

close(bt->pipe_to_child);

close(bt->pipe_from_child);

spew(SPEW_INFO, "waitpid(%d)", bt->child);

int wstatus;

waitpid(bt->child, &wstatus, 0);

spew(SPEW_INFO, "waitpid(%d) done wstatus=%d", bt->child, wstatus);

*bt = errresult;

pthread_mutex_unlock(&backtrace_mutex);

}

static void read_tail_of_line(int fd, char *line, size_t linelen)

// Effect: Read a line of data from fd into line. If the line is too long to

// fit, then put the last characters in so that if we read

// "longfilename/fun.c:25" and linelen is 11 we end up with "e/fun.c:25"

// rather than "longfilena".

// line will always end NUL-terminated.

// Requires: linelen>0. This function will fail unpredictably if linelen==0.

{

for (size_t i = 0; i+1 < linelen; i++) {

char c;

ssize_t rr = read(fd, &c, 1);

if (0 == rr || '\n' == c) {

line[i] = 0;

return;

} else {

line[i] = c;

}

}

while (1) {

char c;

ssize_t rr = read(fd, &c, 1);

if (0 == rr || '\n' == c) {

line[linelen-1] = 0;

return;

} else {

line[linelen-1] = c;

// Shift the string left by 1.o

for (size_t j = 0; j+1 < linelen; j++) {

line[j] = line[j+1];

}

}

}

}

static void spew_backtrace_symbol(struct backtracer *bt,

const void *addr,

const char *from_backtrace_symbol,

size_t spew_level)

{

void *addr_from_base = (void*)((__force size_t)addr

- bt->va_base);

char str_of_ptr[20];

char fun_line[300] = "";

char file_line[300] = "";

if (!bt->broken) {

int len = snprintf(str_of_ptr, sizeof(str_of_ptr),

"%p\n", addr_from_base);

ssize_t wr = write(bt->pipe_to_child, str_of_ptr,

(__force size_t)(ptrdiff_t)len);

int did_write = wr == len;

if (!did_write) {

bt->broken = 1;

} else {

read_tail_of_line(bt->pipe_from_child, fun_line, sizeof(fun_line));

read_tail_of_line(bt->pipe_from_child, file_line, sizeof(file_line));

}

}

if (!from_backtrace_symbol) {

from_backtrace_symbol = "(unknown offset)";

}

const char* function = &fun_line[0];

if (!fun_line[0]) {

function = "(unknown function)";

}

size_t line_number = 0;

const char* filename = &file_line[0];

if (*filename) {

// Find line number by looking for last : in file_line.

char* line_number_start = strrchr(file_line, ':');

if (line_number_start != NULL) {

// Parse the line number

line_number = strtoull(line_number_start + 1, NULL, 10);

if (line_number > 0) {

// Parsed, remove the line number and : from file_line.

*line_number_start = 0;

}

}

} else {

// No filename, substitute backtrace symbol for filename.

filename = from_backtrace_symbol;

from_backtrace_symbol = "";

}

spew_for_backtrace(

spew_level, filename, line_number, function, from_backtrace_symbol);

}

void spew_stored_backtrace(void *const*ptrs, int nptrs, size_t spew_level)

{

char **strings = backtrace_symbols(ptrs, nptrs);

struct backtracer bt = mk_backtracer();

for (int j = 0; j < nptrs; j++) {

spew_backtrace_symbol(&bt,

ptrs[(__force ptrdiff_t)j],

strings ? strings[(__force ptrdiff_t)j] : NULL,

spew_level);

}

backtracer_destroy(&bt);

free(strings);

}

void spew_backtrace(size_t spew_level)

{

if (spew_level < spew_get_level()) return;

void *buffer[100];

int nptrs = backtrace(buffer, (__force int)NELEM(buffer));

/* cannot assert in this file */

if (nptrs >= 0 && nptrs <= (__force int)NELEM(buffer)) {

spew_stored_backtrace(buffer, nptrs, spew_level);

} else {

abort(); // can't happen

}

}

#ifndef __BACKTRACE_H__

#define __BACKTRACE_H__

#include <stdlib.h> // For size_t

/* Fancier backtrace facility.

*/

void spew_backtrace(size_t spew_level);

void spew_stored_backtrace(void *const*ptrs,

// This is an int, rather than a size_t because

// I cannot ASSERT that nelem fits in an int

// inside the spew code.

int nelem,

size_t spew_level);

#endif // !__SPEW_H__