diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..efc19dd --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Golden tests compare golden files (.txt) with \n +*.txt text eol=lf diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..0e193c4 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,63 @@ +name: CI + +on: + push: + pull_request: + +jobs: + staticcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: dominikh/staticcheck-action@v1 + + tests: + strategy: + matrix: + os: + - macos-latest + - ubuntu-latest + - windows-latest + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + # Latest stable version of Go, e.g. 1.20.2 + go-version: 'stable' + + - name: Ensure all files were formatted as per gofmt + if: matrix.os == 'ubuntu-latest' + run: | + [ "$(gofmt -l $(find . -name '*.go') 2>&1)" = "" ] + + - name: install rsync + if: matrix.os == 'ubuntu-latest' + run: | + docker build --pull --no-cache --rm -t=rsync-debian -f testdata/ci-debian.Dockerfile . + docker build --pull --no-cache --rm -t=rsync-fedora -f testdata/ci-fedora.Dockerfile . + + - name: install rsync + if: matrix.os == 'windows-latest' + run: choco install rsync + + - name: run tests (linux) + if: matrix.os == 'ubuntu-latest' + run: | + go test -v ./... + echo "::group::rsync from Debian" + docker run -v $PWD:/usr/src/rsync/ -w /usr/src/rsync rsync-debian go test ./... + echo "::endgroup::" + echo "::group::rsync from Fedora" + docker run -v $PWD:/usr/src/rsync/ -w /usr/src/rsync rsync-fedora go test ./... + echo "::endgroup::" + + - name: run tests (macos) + if: matrix.os == 'macos-latest' + run: sudo go test -v ./... + + - name: run tests (windows) + if: matrix.os == 'windows-latest' + run: go test -v ./... diff --git a/.gitignore b/.gitignore index aaadf73..dc081a7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,2 @@ -# If you prefer the allow list template instead of the deny list, see community template: -# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore -# -# Binaries for programs and plugins -*.exe -*.exe~ -*.dll -*.so -*.dylib - -# Test binary, built with `go test -c` -*.test - -# Code coverage profiles and other test artifacts -*.out -coverage.* -*.coverprofile -profile.cov - -# Dependency directories (remove the comment below to include it) -# vendor/ - -# Go workspace file -go.work -go.work.sum - -# env file -.env - -# Editor/IDE -# .idea/ -# .vscode/ +gon_*.hcl +dist diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..18b3b6c --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,99 @@ +before: + hooks: + - go mod download + - go generate ./... + +builds: +- id: "rsync-linux" + main: "./cmd/gokr-rsync" + binary: "gokr-rsync" + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm + - arm64 + - 386 + +- id: "rsync-windows" + main: "./cmd/gokr-rsync" + binary: "gokr-rsync" + env: + - CGO_ENABLED=0 + goos: + - windows + goarch: + - arm64 + - amd64 + - 386 + +- id: "rsync-macos" + main: "./cmd/gokr-rsync" + binary: "gokr-rsync" + env: + - CGO_ENABLED=0 + goos: + - darwin + goarch: + - arm64 + - amd64 + +archives: +- id: linux + name_template: >- + rsync_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else}}{{ .Arch }}{{ end }} + builds: ["rsync-linux"] + +- id: windows + name_template: >- + rsync_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else}}{{ .Arch }}{{ end }} + builds: ["rsync-windows"] + format: zip + +- id: macos + name_template: >- + rsync_Mac_ + {{- if eq .Arch "amd64" }}Intel + {{- else if eq .Arch "arm64" }}Apple_Silicon + {{- else}}{{ .Arch }}{{ end }} + builds: ["rsync-macos"] + format: zip + +signs: +- id: mac-notarize + ids: [macos] + signature: "${artifact}.dmg" + output: true + cmd: bash + args: + - "-c" + - "gon gon_$(echo ${artifact} | sed 's,^dist/rsync_,,g;s,\\.zip$,,g').hcl" + artifacts: archive + +checksum: + name_template: 'checksums.txt' + +snapshot: + name_template: "{{ .Tag }}-next" + +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' + +release: + github: + owner: gokrazy + name: rsync diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..1cd4a20 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2021 the gokrazy authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of gokrazy nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..967bd40 --- /dev/null +++ b/Makefile @@ -0,0 +1,46 @@ +.PHONY: all run systemd test privileged-test docker raspi mac staticcheck + +all: + CGO_ENABLED=0 go install github.com/runpod/rsync/cmd/... + +staticcheck: + staticcheck ./... + +run: all + sudo ~/go/bin/gokr-rsyncd -modulemap=default=/etc/default + +systemd: all + sudo systemctl stop gokr-rsyncd.socket gokr-rsyncd.service && \ + sudo cp /home/michael/go/bin/gokr-rsyncd /usr/bin/ && \ + sudo cp systemd/gokr-rsyncd.socket systemd/gokr-rsyncd.service /etc/systemd/system/ && \ + sudo systemctl daemon-reload && \ + (sudo systemctl kill -f gokr-rsyncd.service; \ + sudo systemctl restart gokr-rsyncd.socket) + +test: + GOGC=off CGO_ENABLED=0 go test -fullpath ./... + +privileged-test: + GOGC=off CGO_ENABLED=0 sudo go test -fullpath ./integration/interop ./integration/receiver + +docker: + CGO_ENABLED=0 GOBIN=$$PWD/docker go install github.com/runpod/rsync/cmd/gokr-rsyncd + (cd docker && docker build -t=stapelberg/gokrazy-rsync .) + +router7: + GOARCH=amd64 CGO_ENABLED=0 go install github.com/runpod/rsync/cmd/gokr-rsyncd && \ + (ssh router7.lan killall gokr-rsyncd || true) && \ + cp ~/go/bin/gokr-rsyncd /mnt/loop/ && \ + ssh router7.lan /perm/gokr-rsyncd -modulemap distri=/perm/srv/repo.distr1.org/distri/ -listen=10.0.0.1:8730 -monitoring_listen=10.0.0.1:8780 + +raspi: + # -tags nonamespacing + GOARCH=arm64 CGO_ENABLED=0 go install github.com/runpod/rsync/cmd/gokr-rsyncd && \ + ssh gokrazy.lan killall gokr-rsyncd && \ + cp ~/go/bin/linux_arm64/gokr-rsyncd /mnt/loop/ && \ + ssh gokrazy.lan /perm/gokr-rsyncd -modulemap pwd=/gokrazy -listen=:873 + +mac: + GOARCH=arm64 GOOS=darwin CGO_ENABLED=0 go install github.com/runpod/rsync/cmd/gokr-rsyncd && \ + scp ~/go/bin/darwin_arm64/gokr-rsyncd m1a.lan: && \ + ssh m1a.lan ~/gokr-rsyncd -help diff --git a/README.md b/README.md index 82c5244..37ee76d 100644 --- a/README.md +++ b/README.md @@ -1 +1,310 @@ -# rsync \ No newline at end of file +# gokrazy rsync + +[![Go Reference](https://pkg.go.dev/badge/github.com/gokrazy/rsync.svg)](https://pkg.go.dev/github.com/gokrazy/rsync) +[![tests](https://github.com/gokrazy/rsync/actions/workflows/main.yml/badge.svg)](https://github.com/gokrazy/rsync/actions/workflows/main.yml) +[![Sourcegraph](https://sourcegraph.com/github.com/gokrazy/rsync/-/badge.svg)](https://sourcegraph.com/github.com/gokrazy/rsync??badge) + +gokrazy/rsync is [an Rsync program and reusable +library](https://en.wikipedia.org/wiki/Rsync) implemented in the [Go programming +language](https://en.wikipedia.org/wiki/Go_(programming_language)), originally +as part of the [gokrazy Go appliance platform +project](https://gokrazy.org). gokrazy/rsync is (wire-)compatible with the +original "[tridge](https://en.wikipedia.org/wiki/Andrew_Tridgell)" Rsync. + +Various use-cases are supported: + +* The `gokr-rsync` command implements an rsync client and server, which can send + or receive files (all directions supported). +* You can set up a public or private rsync server, in daemon mode or via SSH + (anonymous or authorized). +* You can embed rsync in your program with the [`rsynccmd` + package](https://pkg.go.dev/github.com/gokrazy/rsync/rsynccmd). +* You can implement your own rsync server with the [`rsyncd` + package](https://pkg.go.dev/github.com/gokrazy/rsync/rsyncd). + +Aside from being implemented in Go, a [memory-safe programming +language](https://en.wikipedia.org/wiki/Memory_safety), gokrazy/rsync makes use +of [Go’s traversal-resistant file API (`os.Root`)](https://go.dev/blog/osroot) +and the Linux [Landlock Unprivileged Sandboxing](https://landlock.io/) to [steer +clear of +vulnerabilities](https://michael.stapelberg.ch/posts/2026-05-24-minimal-memory-safe-go-rsync-vulns/). In +server settings, gokrazy/rsync supports mount namespaces and can be further +locked down with systemd’s various hardening options. + +This project accepts contributions as time permits to merge them (best effort). + +## Notable users / use-cases + +* [NVIDIA/OSMA](https://github.com/NVIDIA/OSMO) +* backup of [gokrazy Go appliances with gokrazy/rsync](https://gokrazy.org/userguide/rsync-backups/) +* demo: [rsync file transfer over gRPC transport](https://github.com/stapelberg/rsync-over-grpc) +* Simon Leinen’s presentation: [Go for rsync Diversity at RIPE 90 (2025)](https://ripe90.ripe.net/presentations/127-go-for-rsync-diversity.pdf) + +## How do I know this project won’t eat my data? + +This rsync implementation is not as well-tested as the original “tridge” +implementation from the Samba project. gokrazy/rsync was started in 2021 and +doesn’t have many users yet. + +With that warning out of the way, the rsync protocol uses MD4 checksums over +file contents, so at least your file contents should never be able to be +corrupted. + +There is enough other functionality (delta transfers, file metadata, special +files like symlinks or devices, directory structures, etc.) in the rsync +protocol that provides opportunities for bugs to hide. + +I recommend you carefully check that your transfers work, and please do report +any issues you run into! + +## Existing rsync implementation survey + +| Language | URL | Note | Max Protocol | Server mode? | +|----------|-------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|--------------| +| C | [RsyncProject/rsync](https://github.com/RsyncProject/rsync) (formerly WayneD/rsync) | original “tridge” implementation; I found [older versions](https://github.com/WayneD/rsync/tree/v2.6.1pre2) easier to study | [32](https://github.com/RsyncProject/rsync/blob/v3.4.1/rsync.h#L114) | ✔ yes | +| C | [kristapsdz/openrsync](https://github.com/kristapsdz/openrsync) | OpenBSD, good docs | [27](https://github.com/kristapsdz/openrsync/blob/e54d57f7572381da2b549d39c7968fc79dac8e1d/extern.h#L30) | ✔ yes | +| **Go** | [gokrazy/rsync](https://github.com/gokrazy/rsync) | → you are here ← | [27](https://github.com/gokrazy/rsync/blob/b3b58770b864613551036a2ef2827b74ace77749/internal/rsyncd/rsyncd.go#L317) | ✔ yes 🎉 | +| **Go** | [jbreiding/rsync-go](https://github.com/jbreiding/rsync-go) | rsync algorithm | | ❌ no | +| **Go** | [kaiakz/rsync-os](https://github.com/kaiakz/rsync-os) | only client/receiver | [27](https://github.com/kaiakz/rsync-os/blob/64e84daeabb1fa4d2c7cf766c196306adfba6cb2/rsync/const.go#L4) | ❌ no | +| **Go** | [knight42](https://gist.github.com/knight42/6ad35ce6fbf96519259b43a8c3f37478) | proxy | | ❌ no | +| **Go** | [c4milo/gsync](https://github.com/c4milo/gsync) | | | ❌ no | +| Java | [APNIC-net/repositoryd](https://github.com/APNIC-net/repositoryd) | archived | | ✔ yes | +| Java | [JohannesBuchner/Jarsync](https://github.com/JohannesBuchner/Jarsync/) | archived, [internet draft RFC “The rsync Network Protocol”](https://github.com/JohannesBuchner/Jarsync/blob/master/jarsync/rsync.txt) | | ✔ yes | +| Java | [perlundq/yajsync](https://github.com/perlundq/yajsync#example) | | | ✔ yes | +| C++ | [gilbertchen/acrosync-library](https://github.com/gilbertchen/acrosync-library) | commercial | | ❌ no | +| Rust | [sourcefrog/rsyn](https://github.com/sourcefrog/rsyn#why-do-this) | archived, client, “rsyn is rsync with no c” | [27](https://github.com/sourcefrog/rsyn/blob/2ebbfcfe999fdf2d1a434d8614d07aa93873461b/src/connection.rs#L38) | ❌ no | + +## Getting started + +To serve the `/usr/share/man` directory via rsync on `localhost:8730`, use: + +``` +go install github.com/gokrazy/rsync/cmd/gokr-rsync@latest +gokr-rsync --daemon --gokr.listen=localhost:8730 --gokr.modulemap=man=/usr/share/man +``` + +You can then copy the contents of the current directory with clients such as +`rsync(1)`: + +``` +% rsync -v --archive --port 8730 rsync://localhost/man/ man +receiving file list ... done +created directory man +./ +ar/ +ar/man1/ +[…] +zh_TW/man8/userdel.8.gz +zh_TW/man8/usermod.8.gz + +sent 658.973 bytes received 88.012.067 bytes 3.940.935,11 bytes/sec +total size is 84.504.170 speedup is 0,95 +``` + +…or [`openrsync(1)`](https://github.com/kristapsdz/openrsync), shown doing a +differential update: + +``` +% openrsync -v --archive --port 8730 rsync://localhost/man/ man +openrsync: warning: connect refused: ::1, localhost +Transfer starting: 40202 files +[…] +zh_TW/man8/userdel.8.gz (732 B, 100.0% downloaded) +zh_TW/man8/usermod.8.gz (1.8 KB, 100.0% downloaded) +Transfer complete: 83.93 MB sent, 643.5 KB read, 80.59 MB file size +``` + +## Usage / Setup + + | setup | encrypted | authenticated | private files? | privileges | protocol version | config required | + |-----------------------------------------|-----------|--------------------|------------------------|-----------------------------------------------------------------|------------------|---------------------------------------| + | 1. rsync daemon protocol (TCP port 873) | ❌ no | ⚠ rsync (insecure) | ❌ only world-readable | ✔ dropped + [namespace](#privileged-linux-including-gokrazyorg) | ✔ negotiated | config required | + | 2. anon SSH (daemon) | ✔ yes | ✔ rsync | ❌ only world-readable | ✔ dropped + [namespace](#privileged-linux-including-gokrazyorg) | ✔ negotiated | config required | + | 3. SSH (command) | ✔ yes | ✔ SSH | ✔ yes | ⚠ full user | ⚠ assumed | no config | + | 4. SSH (daemon) | ✔ yes | ✔ SSH (+ rsync) | ✔ yes | ⚠ full user | ✔ negotiated | `~/.config/gokr-rsyncd.toml` required | + +Regarding protocol version “assumed”: the flags to send over the network are +computed *before* starting SSH and hence the remote rsync process. You might +need to specify `--protocol=27` explicitly on the client. Once the connection is +established, both sides *do* negotiate the protocol, though. + +### Setup 1: rsync daemon protocol (TCP port 873) + +Serving rsync daemon protocol on TCP port 873 is only safe where the network +layer ensures trusted communication, e.g. in a local network (LAN), or when +using [Tailscale](https://tailscale.com/) or similar. In untrusted networks, +attackers can eavesdrop on file transfers and possibly even modify file +contents. + +Prefer setup 2 instead. + +Example: +* Server: `gokr-rsync --daemon --gokr.modulemap=module=/srv/rsync-module` +* Client: `rsync rsync://webserver/module/path` + +### Setup 2: anon SSH (daemon) + +This setup is well suited for serving world-readable files without +authentication. + +Example: +* Server: `gokr-rsync --daemon --gokr.modulemap=module=/srv/rsync-module --gokr.anonssh_listen=:22873` +* Client: `rsync -e ssh rsync://webserver/module/path` + + +### Setup 3: SSH (command) + +This setup is well suited for interactive one-off transfers or regular backups, +and uses SSH for both encryption and authentication. + +Note that because `gokr-rsync` is invoked with user privileges (not root +privileges), it cannot do [namespacing](#privileged-linux-including-gokrazyorg) +and hence retains more privileges. When serving public data, it is generally +preferable to use setup 2 instead. + +Note that `rsync(1)` assumes the server process understands all flags that it +sends, i.e. is running the same version on client and server, or at least a +compatible-enough version. You can either specify `--protocol=27` on the client, +or use setup 4, which negotiates the protocol version, side-stepping possible +compatibility gaps between rsync clients and `gokr-rsync`. + +Example: +* Server will be started via SSH +* Client: `rsync --rsync-path=gokr-rsync webserver:path` + +### Setup 4: SSH (daemon) + +This setup is more reliable than setup 3 because the rsync protocol version will +be negotiated between client and server. This setup is slightly inconvenient +because it requires a config file to be present on the server in +`~/.config/gokr-rsyncd.toml`. + +Note that this mode of operation is only implemented by the original “trigde” +rsync, not in openrsync. Apple started shipping openrsync with macOS 15 Sequoia. +For a while, `/usr/libexec/rsync/rsync.samba` was still available, but on more +recent macOS versions you need to use homebrew or Nix to get tridge rsync. + +Example: +* Server will be started via SSH +* Client: `rsync -e ssh --rsync-path=gokr-rsync rsync://webserver/module/path` + +## Limitations + +### Bandwidth + +In my tests, `gokr-rsync` can easily transfer data at > 6 Gbit/s. The current +bottleneck is the MD4 algorithm itself (not sure whether in the “tridge” rsync +client, or in `gokr-rsync`). Implementing support for more recent protocol +versions would help here, as these include hash algorithm negotiation with more +recent choices. + +### Protocol related limitations + +* xattrs (including acls) was introduced in rsync protocol 30, so is currently + not supported. + +## Supported environments and privilege dropping + +Supported environments: + +1. systemd (Linux) +1. privileged Linux (running as `root`, or in a user namespace) +1. privileged non-Linux +1. unprivileged Linux, Mac or Windows + +In all environments, the default instructions will take care that: + +* `gokr-rsync` uses Go’s [Traversal-resistant `os.Root` file + APIs](https://go.dev/blog/osroot) to restrict all file access to the specified + paths. +* (On privileged Linux only) Only configured rsync modules from the host file + system are mounted (**read-only**, unless the module is `writable`) into a + Linux mount namespace for `gokr-rsync`, to guard against data modification and + data exfiltration. +* (On Linux only) File system access is restricted using the + [Landlock](https://docs.kernel.org/userspace-api/landlock.html) Linux kernel + security module, which works similar to OpenBSD’s + [`unveil(2)`](https://man.openbsd.org/unveil.2) API. +* (On privileged environments) `gokr-rsync` drops privileges to user `nobody`, + to limit the scope of what an attacker can do when exploiting a vulnerability. + +Known gaps: + +* `gokr-rsync` does not guard against denial of service attacks, i.e. consuming + too many resources (connections, bandwidth, CPU, …). + * See also [Per-IP rate limiting with + iptables](https://making.pusher.com/per-ip-rate-limiting-with-iptables/). + + +### systemd (unprivileged) + +We provide [a `gokr-rsyncd.socket` and `gokr-rsyncd.service` +file](https://github.com/gokrazy/rsync/tree/main/systemd/) for systemd. These +files enables most of systemd’s security features. You can check by running +`systemd-analyze security gokr-rsyncd.service`, which should result in an +exposure level of “0.2 SAFE” as of systemd 249 (September 2021). + +First, configure your server flags by creating a systemd service override file: + +```shell +systemctl edit gokr-rsyncd.service +``` + +In the opened editor, change the file to: +``` +[Service] +ExecStart= +ExecStart=/usr/bin/gokr-rsync --daemon --gokr.modulemap=pwd=/etc/tmpfiles.d +``` + +Close the editor and install the service using: + +```shell +systemctl enable --now gokr-rsyncd.socket +``` + +Additional hardening recommendations: + +* Restrict which IP addresses are allowed to connect to your rsync server, for example: + * using iptables or nftables on your host system + * using [`gokr-rsync`’s built-in IP allow/deny mechanism](https://github.com/gokrazy/rsync/commit/322543c7c9ee5f9b2128b6f7ccc931d05ae21df1) + * using [systemd’s `IPAddressDeny` and `IPAddressAllow`](https://manpages.debian.org/systemd.resource-control.5) in `gokr-rsyncd.socket` +* To reduce the impact of Denial Of Service attacks, you can restrict resources + with systemd, see [Managing + Resources](http://0pointer.de/blog/projects/resources.html). +* To hide system directories not relevant to any rsync module, use [systemd’s + `TemporaryFileSystem=` and + `BindReadOnlyPaths=`](https://manpages.debian.org/systemd.exec.5) directives + as described in [Use TemporaryFileSystem to hide files or directories from + systemd + services](https://www.sherbers.de/use-temporaryfilesystem-to-hide-files-or-directories-from-systemd-services/). Note + that you [may need to disable `ProtectSystem=strict` due to a + bug](https://github.com/systemd/systemd/issues/18999). + +### privileged Linux (including gokrazy.org) + +When started as `root` on Linux, `gokr-rsync` will create a [Linux mount +namespace](https://manpages.debian.org/mount_namespaces.7), mount all configured +rsync modules read-only into the namespace, then change into the namespace using +[`chroot(2)`](https://manpages.debian.org/chroot.2) and drop privileges using +[`setuid(2)`](https://manpages.debian.org/setuid.2). + +**Tip:** you can verify which file system objects the daemon process can see by +using `ls -l /proc/$(pidof gokr-rsync)/root/`. + +Additional hardening recommendations: + +* Restrict which IP addresses are allowed to connect to your rsync server, for example: + * using iptables or nftables on your host system + * using [`gokr-rsync`’s built-in IP allow/deny mechanism](https://github.com/gokrazy/rsync/commit/322543c7c9ee5f9b2128b6f7ccc931d05ae21df1) + +### privileged non-Linux (e.g. Mac) + +When started as `root` on non-Linux (e.g. Mac), `gokr-rsync` will drop +privileges using [`setuid(2)`](https://manpages.debian.org/setuid.2). + +### unprivileged with write permission (e.g. from a shell) + +To prevent accidental misconfiguration, `gokr-rsync` refuses to start when it +detects that it has write permission in any configured rsync module. + diff --git a/closer.go b/closer.go new file mode 100644 index 0000000..57a9bad --- /dev/null +++ b/closer.go @@ -0,0 +1,21 @@ +package rsync + +import "io" + +// BothCloser is a wrapper around a ReadCloser and WriteCloser that provides its +// own Close() function which calls Close() on both, the WriteCloser and +// ReadCloser, in order, and returns the first non-nil error. +type BothCloser struct { + io.ReadCloser + io.WriteCloser +} + +// Close implements io.Closer. +func (b *BothCloser) Close() error { + wcErr := b.WriteCloser.Close() + rcErr := b.ReadCloser.Close() + if wcErr != nil { + return wcErr + } + return rcErr +} diff --git a/cmd/gokr-rsync/rsync.go b/cmd/gokr-rsync/rsync.go new file mode 100644 index 0000000..d85c687 --- /dev/null +++ b/cmd/gokr-rsync/rsync.go @@ -0,0 +1,25 @@ +// Tool gokr-rsync is an rsync Go implementation. +package main + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "github.com/runpod/rsync/rsynccmd" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + cmd := rsynccmd.Command(os.Args[0], os.Args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if _, err := cmd.Run(ctx); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/gokr-rsyncd/rsyncd.go b/cmd/gokr-rsyncd/rsyncd.go new file mode 100644 index 0000000..3420f9b --- /dev/null +++ b/cmd/gokr-rsyncd/rsyncd.go @@ -0,0 +1,29 @@ +// Tool gokr-rsyncd is an old name for gokr-rsync. +// +// Please update your setup to install/use gokr-rsync directly instead. +// +// This program will be removed in a future release. +package main + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "github.com/runpod/rsync/rsynccmd" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + cmd := rsynccmd.Command(os.Args[0], os.Args[1:]...) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if _, err := cmd.Run(ctx); err != nil { + log.Fatal(err) + } +} diff --git a/consts.go b/consts.go new file mode 100644 index 0000000..2a06440 --- /dev/null +++ b/consts.go @@ -0,0 +1,36 @@ +package rsync + +// rsync.h +const ( + XMIT_TOP_DIR = (1 << 0) + XMIT_SAME_MODE = (1 << 1) + XMIT_EXTENDED_FLAGS = (1 << 2) + XMIT_SAME_RDEV_pre28 = XMIT_EXTENDED_FLAGS /* Only in protocols < 28 */ + XMIT_SAME_UID = (1 << 3) + XMIT_SAME_GID = (1 << 4) + XMIT_SAME_NAME = (1 << 5) + XMIT_LONG_NAME = (1 << 6) + XMIT_SAME_TIME = (1 << 7) + XMIT_SAME_RDEV_MAJOR = (1 << 8) + XMIT_HAS_IDEV_DATA = (1 << 9) + XMIT_SAME_DEV = (1 << 10) + XMIT_RDEV_MINOR_IS_SMALL = (1 << 11) +) + +// as per /usr/include/bits/stat.h: +const ( + S_IFMT = 0o0170000 // bits determining the file type + S_IFDIR = 0o0040000 // Directory + S_IFCHR = 0o0020000 // Character device + S_IFBLK = 0o0060000 // Block device + S_IFREG = 0o0100000 // Regular file + S_IFIFO = 0o0010000 // FIFO + S_IFLNK = 0o0120000 // Symbolic link + S_IFSOCK = 0o0140000 // Socket +) + +// ProtocolVersion defines the currently implemented rsync protocol +// version. Protocol version 27 seems to be the safest bet for wide +// compatibility: version 27 was introduced by rsync 2.6.0 (released 2004), and +// is supported by openrsync and rsyn. +const ProtocolVersion = 27 diff --git a/doc.go b/doc.go new file mode 100644 index 0000000..af8c7f5 --- /dev/null +++ b/doc.go @@ -0,0 +1,9 @@ +// Package rsync (gokrazy/rsync) contains a native Go rsync implementation that +// supports sending and receiving files as client or server, compatible with the +// original tridge rsync (from the samba project) or openrsync (used on OpenBSD +// and macOS 15+). +// +// The only component currently is gokr-rsyncd, a read-only rsync daemon +// sender-only Go implementation of rsyncd. rsync daemon is a custom +// (un-standardized) network protocol, running on port 873 by default. +package rsync diff --git a/fsroot.go b/fsroot.go new file mode 100644 index 0000000..ce481a8 --- /dev/null +++ b/fsroot.go @@ -0,0 +1,5 @@ +//go:build !windows + +package rsync + +const FileSystemRoot = "/" diff --git a/fsroot_windows.go b/fsroot_windows.go new file mode 100644 index 0000000..7ced9f4 --- /dev/null +++ b/fsroot_windows.go @@ -0,0 +1,5 @@ +//go:build windows + +package rsync + +const FileSystemRoot = "\\\\?\\" diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..420089a --- /dev/null +++ b/go.mod @@ -0,0 +1,20 @@ +module github.com/runpod/rsync + +go 1.25.0 + +require ( + github.com/BurntSushi/toml v1.6.0 + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf + github.com/google/go-cmp v0.7.0 + github.com/google/renameio/v2 v2.0.2 + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 + github.com/mmcloughlin/md4 v0.1.2 + golang.org/x/crypto v0.46.0 + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.39.0 +) + +require ( + github.com/landlock-lsm/go-landlock v0.0.0-20250303204525-1544bccde3a3 + kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..df4be4c --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/renameio/v2 v2.0.2 h1:qKZs+tfn+arruZZhQ7TKC/ergJunuJicWS6gLDt/dGw= +github.com/google/renameio/v2 v2.0.2/go.mod h1:OX+G6WHHpHq3NVj7cAOleLOwJfcQ1s3uUJQCrr78SWo= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/landlock-lsm/go-landlock v0.0.0-20250303204525-1544bccde3a3 h1:zcMi8R8vP0WrrXlFMNUBpDy/ydo3sTnCcUPowq1XmSc= +github.com/landlock-lsm/go-landlock v0.0.0-20250303204525-1544bccde3a3/go.mod h1:RSub3ourNF8Hf+swvw49Catm3s7HVf4hzdFxDUnEzdA= +github.com/mmcloughlin/md4 v0.1.2 h1:kGYl+iNbxhyz4u76ka9a+0TXP9KWt/LmnM0QhZwhcBo= +github.com/mmcloughlin/md4 v0.1.2/go.mod h1:AAxFX59fddW0IguqNzWlf1lazh1+rXeIt/Bj49cqDTQ= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 h1:HsB2G/rEQiYyo1bGoQqHZ/Bvd6x1rERQTNdPr1FyWjI= +kernel.org/pub/linux/libs/security/libcap/psx v1.2.70/go.mod h1:+l6Ee2F59XiJ2I6WR5ObpC1utCQJZ/VLsEbQCD8RG24= diff --git a/integration/errors/errors_test.go b/integration/errors/errors_test.go new file mode 100644 index 0000000..f29e7d8 --- /dev/null +++ b/integration/errors/errors_test.go @@ -0,0 +1,147 @@ +package errors_test + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsynctest" +) + +func TestErrors(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + + dest := filepath.Join(tmp, "dest") + + // We configure an rsync module with a non-existant path to trigger an + // error. Removing read permission from a file is not sufficient because + // that does not actually trigger an error! See TestNoReadPermission. + nonExistant := filepath.Join(tmp, "non/existant") + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(nonExistant)) + + // sync into dest dir + var buf bytes.Buffer + rsync := exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost/interop/", // copy contents of interop + //source+"/", // sync from local directory + filepath.Base(dest)) // directly into dest (relative to rsync.Dir) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = &buf + rsync.Stderr = &buf + if err := rsync.Run(); err == nil { + t.Fatalf("rsync unexpectedly did not return with an error exit code, output:\n%s", buf.String()) + } + + output := buf.String() + t.Logf("output:\n%s\n(end of output)", output) + if want := "module path is not accessible"; !strings.Contains(output, want) { + t.Fatalf("rsync output unexpectedly did not contain %q:\n%s", want, output) + } +} + +func TestNoSuchModule(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + + dest := filepath.Join(tmp, "dest") + + // start a server to sync from + srv := rsynctest.New(t, nil) + + // sync into dest dir + var buf bytes.Buffer + rsync := exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost/requesting-nonsense/", // copy contents of interop + //source+"/", // sync from local directory + filepath.Base(dest)) // directly into dest (relative to rsync.Dir) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = &buf + rsync.Stderr = &buf + if err := rsync.Run(); err == nil { + t.Fatalf("rsync unexpectedly did not return with an error exit code, output:\n%s", buf.String()) + } + + output := buf.String() + if want := "Unknown module"; !strings.Contains(output, want) { + t.Fatalf("rsync output unexpectedly did not contain %q:\n%s", want, output) + } +} + +func TestNoReadPermission(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + // create files in source to be copied + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + dummy := filepath.Join(source, "dummy") + if err := os.WriteFile(dummy, []byte("dummy"), 0644); err != nil { + t.Fatal(err) + } + other := filepath.Join(source, "other") + want := []byte("other file contents") + if err := os.WriteFile(other, want, 0644); err != nil { + t.Fatal(err) + } + + // Remove read permission to trigger an error for one of the requested files. + if err := os.Chmod(dummy, 0); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + var buf bytes.Buffer + rsync := exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost/interop/", // copy contents of interop + filepath.Base(dest)) // directly into dest (relative to rsync.Dir) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = &buf + rsync.Stderr = &buf + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if os.Getuid() > 0 { + // uid 0 can read the file despite chmod(0), so skip this check: + + if _, err := os.ReadFile(filepath.Join(dest, "dummy")); err == nil { + t.Fatalf("dummy file unexpectedly created in the destination") + } + } + + got, err := os.ReadFile(filepath.Join(dest, "other")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } +} diff --git a/integration/flist/flist_test.go b/integration/flist/flist_test.go new file mode 100644 index 0000000..282e213 --- /dev/null +++ b/integration/flist/flist_test.go @@ -0,0 +1,37 @@ +package flist_test + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/rsyncd" +) + +// rsynctest.go:282: length 280063 exceeds max message size (262144) +func TestLargeFileList(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + + source := filepath.Join(tmp, "source") + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + for i := range 5000 { + fn := fmt.Sprintf("file_with_long_name_number_%04d", i) + if err := os.WriteFile(filepath.Join(source, fn), []byte("dummy"), 0644); err != nil { + t.Fatal(err) + } + } + dest := filepath.Join(tmp, "dest") + + // start a server to sync from + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "interop", + Path: source, + }) + srv.RunClient(t, []string{"-aH"}, "./", []string{dest}) +} diff --git a/integration/fsfs/fsfs_test.go b/integration/fsfs/fsfs_test.go new file mode 100644 index 0000000..452e4c5 --- /dev/null +++ b/integration/fsfs/fsfs_test.go @@ -0,0 +1,324 @@ +package fsfs_test + +import ( + "log" + "os" + "os/exec" + "path/filepath" + "testing" + "testing/fstest" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" + "github.com/runpod/rsync/rsyncd" +) + +func TestMain(m *testing.M) { + if err := rsynctest.CommandMain(m); err != nil { + log.Fatal(err) + } +} + +func TestMapFS(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + dest := filepath.Join(tmp, "dest") + + memfs := fstest.MapFS{ + "hello.txt": &fstest.MapFile{ + Data: []byte("world"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + "bye.txt": &fstest.MapFile{ + Data: []byte("moon"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + } + + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "memfs", + FS: memfs, + }) + args := []string{"-av"} + srv.RunClient(t, args, "./", []string{dest + "/"}) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("hello.txt: unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + want := []byte("moon") + got, err := os.ReadFile(filepath.Join(dest, "bye.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("world.txt: unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + // Restore write permission so that t.TempDir() cleanup succeeds + if err := os.Chmod(dest, 0755); err != nil { + t.Fatal(err) + } +} + +func TestMapFSSubdirNested(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + dest := filepath.Join(tmp, "dest") + + memfs := fstest.MapFS{ + "treasure/expensive/hello.txt": &fstest.MapFile{ + Data: []byte("world"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + } + + srv := rsynctest.New(t, []rsyncd.Module{ + { + Name: "memfs", + FS: memfs, + }, + }) + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + []string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + "rsync://localhost/memfs/treasure/expensive/", // copy contents of expensive + }, + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("hello.txt: unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + // Restore write permission so that t.TempDir() cleanup succeeds + if err := os.Chmod(dest, 0755); err != nil { + t.Fatal(err) + } +} + +func TestMapFSSubdirNestedNoSlash(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + dest := filepath.Join(tmp, "dest") + + memfs := fstest.MapFS{ + "treasure/expensive/hello.txt": &fstest.MapFile{ + Data: []byte("world"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + } + + srv := rsynctest.New(t, []rsyncd.Module{ + { + Name: "memfs", + FS: memfs, + }, + }) + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + []string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + "rsync://localhost/memfs/treasure/expensive", // copy expensive + }, + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "expensive", "hello.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("hello.txt: unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + // Restore write permission so that t.TempDir() cleanup succeeds + if err := os.Chmod(dest, 0755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(dest, "expensive"), 0755); err != nil { + t.Fatal(err) + } +} + +func TestMapFSSingleFile(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + dest := filepath.Join(tmp, "dest") + + memfs := fstest.MapFS{ + "expensive/hello.txt": &fstest.MapFile{ + Data: []byte("world"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + } + + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "memfs", + FS: memfs, + }) + args := []string{"-av"} + srv.RunClient(t, args, "./expensive/hello.txt", []string{dest + "/"}) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("hello.txt: unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + // Restore write permission so that t.TempDir() cleanup succeeds + if err := os.Chmod(dest, 0755); err != nil { + t.Fatal(err) + } +} + +func TestMapFSErrors(t *testing.T) { + t.Parallel() + + memfs := fstest.MapFS{ + "hello.txt": &fstest.MapFile{ + Data: []byte("world"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + } + _, err := rsyncd.NewServer([]rsyncd.Module{ + { + Name: "test", + FS: memfs, + Writable: true, + }, + }) + if err == nil { + t.Fatal("expected error for writable FS module, got nil") + } + + _, err = rsyncd.NewServer([]rsyncd.Module{ + { + Name: "test", + Path: "/", + FS: memfs, + }, + }) + if err == nil { + t.Fatal("expected error for module with Path and FS, got nil") + } +} + +func TestMapFSLargeFile(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large.bin") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + large := rsynctest.ConstructLargeDataFile(headPattern, bodyPattern, endPattern) + + memfs := fstest.MapFS{ + "hello.txt": &fstest.MapFile{ + Data: []byte("world"), + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + "large.bin": &fstest.MapFile{ + Data: large, + Mode: 0o644, + ModTime: rsynctest.GosPublicRelease, + }, + } + + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "memfs", + FS: memfs, + }) + args := []string{"-av"} + firstStats := srv.RunClient(t, args, "./", []string{dest}) + t.Logf("firstStats: %+v", firstStats) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("hello.txt: unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + + // Change the middle of the large data file: + bodyPattern = []byte{0x66} + // modify the large data file in memory + large = rsynctest.ConstructLargeDataFile(headPattern, bodyPattern, endPattern) + memfs["large.bin"].Data = large + + incrementalStats := srv.RunClient(t, args, "./", []string{dest}) + t.Logf("incrementalStats: %+v", incrementalStats) + if got, want := incrementalStats.Written, int64(2*1024*1024); got >= want { + t.Fatalf("rsync unexpectedly transferred more data than needed: got %d, want < %d", got, want) + } + + // Restore write permission so that t.TempDir() cleanup succeeds + if err := os.Chmod(dest, 0755); err != nil { + t.Fatal(err) + } +} diff --git a/integration/interop/interop_test.go b/integration/interop/interop_test.go new file mode 100644 index 0000000..a42e0b5 --- /dev/null +++ b/integration/interop/interop_test.go @@ -0,0 +1,919 @@ +package interop_test + +import ( + "bytes" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/BurntSushi/toml" + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" + "github.com/runpod/rsync/rsyncd" +) + +func TestMain(m *testing.M) { + if err := rsynctest.CommandMain(m); err != nil { + log.Fatal(err) + } +} + +func TestRsyncVersion(t *testing.T) { + // This function is not an actual test, just used to include the rsync + // version in test output. + rsync := exec.Command(rsynctest.AnyRsync(t), "--version") + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } +} + +func TestTridgeRsyncVersion(t *testing.T) { + // This function is not an actual test, just used to include the rsync + // version in test output. + rsyncBin := rsynctest.TridgeOrGTFO(t, "--version") + tridgeRsync := exec.Command(rsyncBin, "--version") + tridgeRsync.Stdout = testlogger.New(t) + tridgeRsync.Stderr = testlogger.New(t) + if err := tridgeRsync.Run(); err != nil { + t.Fatalf("%v: %v", tridgeRsync.Args, err) + } +} + +func TestModuleListingServer(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: add reason") + + tmp := t.TempDir() + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(tmp)) + + // request module list + var buf bytes.Buffer + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost") + rsync.Stdout = &buf + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + output := buf.String() + if want := "interop\tinterop"; !strings.Contains(output, want) { + t.Fatalf("rsync output unexpectedly did not contain %q:\n%s", want, output) + } +} + +func TestModuleListingClient(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(tmp)) + + // request module list + args := []string{ + "gokr-rsync", + "-aH", + "rsync://localhost:" + srv.Port + "/", + } + stdout, _ := rsynctest.Output(t, args...) + + if want := "interop\tinterop"; !strings.Contains(string(stdout), want) { + t.Fatalf("rsync output unexpectedly did not contain %q:\n%s", want, string(stdout)) + } +} + +func TestModuleListingClientPort(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(tmp)) + + // request module list + args := []string{ + "gokr-rsync", + "-aH", + "--port=" + srv.Port, + "rsync://localhost/", + } + stdout, _ := rsynctest.Output(t, args...) + + if want := "interop\tinterop"; !strings.Contains(string(stdout), want) { + t.Fatalf("rsync output unexpectedly did not contain %q:\n%s", want, string(stdout)) + } +} + +func TestModuleContentsListingDirs(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + + // create files in source to be copied + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + dummy := filepath.Join(source, "dummy") + want := []byte("heyo") + if err := os.WriteFile(dummy, want, 0644); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(tmp)) + + // request module content listing + rsync := exec.Command(rsynctest.AnyRsync(t), + "--dirs", + "--list-only", + "--port="+srv.Port, + "rsync://localhost/interop") + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } +} + +func TestInterop(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + // create files in source to be copied + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + dummy := filepath.Join(source, "dummy") + want := []byte("heyo") + if err := os.WriteFile(dummy, want, 0644); err != nil { + t.Fatal(err) + } + + linkToDummy := filepath.Join(source, "link_to_dummy") + if err := os.Symlink("dummy", linkToDummy); err != nil { + t.Fatal(err) + } + + if os.Getuid() == 0 { + rsynctest.CreateDummyDeviceFiles(t, source) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // { + // config := filepath.Join(tmp, "rsyncd.conf") + // rsyncdConfig := ` + // use chroot = no + // # 0 = no limit + // max connections = 0 + // pid file = ` + tmp + `/rsyncd.pid + // exclude = lost+found/ + // transfer logging = yes + // timeout = 900 + // ignore nonreadable = yes + // dont compress = *.gz *.tgz *.zip *.z *.Z *.rpm *.deb *.bz2 *.zst + + // [interop] + // path = /home/michael/i3/docs + // #` + source + ` + // comment = interop + // read only = yes + // list = true + + // ` + // if err := os.WriteFile(config, []byte(rsyncdConfig), 0644); err != nil { + // t.Fatal(err) + // } + // srv := exec.Command(rsynctest.AnyRsync(t), + // "--daemon", + // "--config="+config, + // "--verbose", + // "--address=localhost", + // "--no-detach", + // "--port=8730") + // srv.Stdout = os.Stdout + // srv.Stderr = os.Stderr + // if err := srv.Start(); err != nil { + // t.Fatal(err) + // } + // go func() { + // if err := srv.Wait(); err != nil { + // t.Error(err) + // } + // }() + // defer srv.Process.Kill() + // + // time.Sleep(1 * time.Second) + // } + + // dry run (slight differences in protocol) + rsync := exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "--dry-run", + "rsync://localhost/interop/", // copy contents of interop + //source+"/", // sync from local directory + filepath.Base(dest)) // directly into dest + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + // sync into dest dir + rsync = exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost/interop/", // copy contents of interop + //source+"/", // sync from local directory + filepath.Base(dest)) // directly into dest + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + { + got, err := os.ReadFile(filepath.Join(dest, "dummy")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + // Symlinks work differently on Windows. + if runtime.GOOS != "windows" { + got, err := os.Readlink(filepath.Join(dest, "link_to_dummy")) + if err != nil { + t.Fatal(err) + } + if want := "dummy"; got != want { + t.Fatalf("unexpected symlink target: got %q, want %q", got, want) + } + } + + if os.Getuid() == 0 { + rsynctest.VerifyDummyDeviceFiles(t, source, dest) + } + + // Run rsync again. This should not modify any files, but will result in + // rsync sending sums to the sender. + rsync = exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + // TODO: should this be --checksum instead? + "--ignore-times", // disable rsync’s “quick check” + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost/interop/", // copy contents of interop + //source+"/", // sync from local directory + filepath.Base(dest)) // directly into dest + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + +} + +func createSourceFiles(t *testing.T) (string, string, string) { + t.Helper() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + // create files in source to be copied + subDirs := []string{"expensive", "cheap"} + for _, subdir := range subDirs { + dummy := filepath.Join(source, subdir, "dummy") + if err := os.MkdirAll(filepath.Dir(dummy), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dummy, []byte(subdir), 0644); err != nil { + t.Fatal(err) + } + } + + return tmp, source, dest +} + +func sourcesArgs(t *testing.T) []string { + if strings.HasPrefix(rsynctest.RsyncVersion(t), "3.") { + // rsync 3.0.0 (March 2008) introduced multiple source args. + return []string{ + "rsync://localhost/interop/expensive/", // copy contents of interop + "rsync://localhost/interop/cheap", // copy cheap directory + } + } + // Older rsync only supports a single source arg. + return []string{ + "rsync://localhost/interop/expensive/", // copy contents of interop + } +} + +func sourceFullySyncedTo(t *testing.T, dest string) error { + { + want := []byte("expensive") + got, err := os.ReadFile(filepath.Join(dest, "dummy")) + if err != nil { + return err + } + if diff := cmp.Diff(want, got); diff != "" { + return fmt.Errorf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + if !strings.HasPrefix(rsynctest.RsyncVersion(t), "3.") { + return nil + } + + { + want := []byte("cheap") + got, err := os.ReadFile(filepath.Join(dest, "cheap", "dummy")) + if err != nil { + return err + } + if diff := cmp.Diff(want, got); diff != "" { + return fmt.Errorf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + return nil +} + +func TestInteropSingleFile(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + []string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + "rsync://localhost/interop/expensive/dummy", + }, + filepath.Base(dest)+"/")...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + { + want := []byte("expensive") + got, err := os.ReadFile(filepath.Join(dest, "dummy")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} + +// Like TestInteropSingleFile, but without a trailing slash +// in the dest argument (no subdirectory to create). +func TestInteropSingleFileNoSlash(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + []string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + "rsync://localhost/interop/expensive/dummy", + }, + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + { + want := []byte("expensive") + got, err := os.ReadFile(dest) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} + +func TestInteropSubdir(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + append([]string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + }, sourcesArgs(t)...), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := sourceFullySyncedTo(t, dest); err != nil { + t.Fatal(err) + } +} + +func TestInteropSubdirNested(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + // Move the "expensive" directory one level deeper: + // now "treasure/expensive". + treasure := filepath.Join(source, "treasure") + if err := os.MkdirAll(treasure, 0755); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Join(source, "expensive"), filepath.Join(treasure, "expensive")); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + []string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + "rsync://localhost/interop/treasure/expensive/", // copy contents of expensive + }, + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + want := []byte("expensive") + got, err := os.ReadFile(filepath.Join(dest, "dummy")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } +} + +func TestInteropSubdirExclude(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + append([]string{ + // "--debug=all4", + "--archive", + // TODO: implement support for include rules + //"-f", "+ *.o", + // NOTE: Using -f is the more modern replacement + // for using --exclude like so: + //"--exclude=dummy", + "-f", "- expensive", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + }, "rsync://localhost/interop/"), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + expensiveFn := filepath.Join(dest, "expensive", "dummy") + if _, err := os.ReadFile(expensiveFn); !os.IsNotExist(err) { + t.Fatalf("ReadFile(%s) did not return -ENOENT, but %v", expensiveFn, err) + } + cheapFn := filepath.Join(dest, "cheap", "dummy") + if _, err := os.ReadFile(cheapFn); err != nil { + t.Fatalf("ReadFile(%s): %v", cheapFn, err) + } +} + +func TestInteropSubdirExcludeGokrazy(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + args := []string{ + "gokr-rsync", + "-f", "- expensive", + "-avH", + "rsync://localhost:" + srv.Port + "/interop/", + dest, + } + rsynctest.Run(t, args...) + + expensiveFn := filepath.Join(dest, "expensive", "dummy") + if _, err := os.ReadFile(expensiveFn); !os.IsNotExist(err) { + t.Fatalf("ReadFile(%s) did not return -ENOENT, but %v", expensiveFn, err) + } + cheapFn := filepath.Join(dest, "cheap", "dummy") + if _, err := os.ReadFile(cheapFn); err != nil { + t.Fatalf("ReadFile(%s): %v", cheapFn, err) + } +} + +func TestInteropSubdirExcludeMultipleNested(t *testing.T) { + t.Parallel() + + _, source, dest := createSourceFiles(t) + + nested := filepath.Join(source, "nested") + + // create files in source to be copied + subDirs := []string{"nested-expensive", "nested-cheap"} + for _, subdir := range subDirs { + dummy := filepath.Join(nested, subdir, "dummy") + if err := os.MkdirAll(filepath.Dir(dummy), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dummy, []byte(subdir), 0644); err != nil { + t.Fatal(err) + } + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + append([]string{ + // "--debug=all4", + "--archive", + // TODO: implement support for include rules + //"-f", "+ *.o", + // NOTE: Using -f is the more modern replacement + // for using --exclude like so: + //"--exclude=dummy", + "-f", "- nested/nested-expensive", + "-f", "- nested/nested-cheap", + "-v", "-v", "-v", "-v", + "--port=" + srv.Port, + }, "rsync://localhost/interop/"), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + expensiveFn := filepath.Join(dest, "nested", "nested-expensive", "dummy") + if _, err := os.ReadFile(expensiveFn); !os.IsNotExist(err) { + t.Fatalf("ReadFile(%s) did not return -ENOENT, but %v", expensiveFn, err) + } + cheapFn := filepath.Join(dest, "nested", "nested-cheap", "dummy") + if _, err := os.ReadFile(cheapFn); !os.IsNotExist(err) { + t.Fatalf("ReadFile(%s) did not return -ENOENT, but %v", cheapFn, err) + } +} + +func TestInteropRemoteCommand(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("stdin not supported on Windows") + } + + _, source, dest := createSourceFiles(t) + + sourcesArgs := []string{ + "localhost:" + source + "/expensive/", // copy contents of interop + } + if strings.HasPrefix(rsynctest.RsyncVersion(t), "3.") { + sourcesArgs = append(sourcesArgs, ":"+source+"/cheap") // copy cheap directory + } + + // sync into dest dir + rsync := exec.Command(rsynctest.AnyRsync(t), + append( + append([]string{ + // "--debug=all4", + "--archive", + "--protocol=27", + "-v", "-v", "-v", "-v", + "-e", `"` + os.Args[0] + `"`, + }, sourcesArgs...), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := sourceFullySyncedTo(t, dest); err != nil { + t.Fatal(err) + } +} + +func TestInteropRemoteDaemon(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("stdin not supported on Windows") + } + + rsyncBin := rsynctest.TridgeOrGTFO(t, "https://github.com/runpod/rsync/issues/33") + + tmp, source, dest := createSourceFiles(t) + + homeDir := filepath.Join(tmp, "home") + // Use os.Setenv so that the os.UserConfigDir() call below returns the + // correct path. + os.Setenv("HOME", homeDir) + os.Setenv("XDG_CONFIG_HOME", homeDir+"/.config") + + { + // in remote daemon mode, rsync needs a config file, so we create one and + // set the HOME environment variable such that gokr-rsyncd will pick it up. + cfg := rsyncdconfig.Config{ + Modules: []rsyncd.Module{ + {Name: "interop", Path: source}, + }, + } + configDir, err := os.UserConfigDir() + if err != nil { + t.Fatal(err) + } + configPath := filepath.Join(configDir, "gokr-rsyncd.toml") + if err := os.MkdirAll(filepath.Dir(configPath), 0755); err != nil { + t.Fatal(err) + } + f, err := os.Create(configPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + if err := toml.NewEncoder(f).Encode(&cfg); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + } + + // sync into dest dir + rsync := exec.Command(rsyncBin, + append( + append([]string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "-e", `"` + os.Args[0] + `"`, + }, sourcesArgs(t)...), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + // TODO: does os.Environ() reflect changes by os.Setenv()? + rsync.Env = append(os.Environ(), + "HOME="+homeDir, + "XDG_CONFIG_HOME="+homeDir+"/.config") + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := sourceFullySyncedTo(t, dest); err != nil { + t.Fatal(err) + } +} + +func TestInteropRemoteDaemonSSH(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("/dev/null not supported on Windows") + } + + // ensure the user running the tests (root when doing the privileged run!) + // has an SSH private key: + privKeyPath := filepath.Join(t.TempDir(), "ssh_private_key") + genKey := exec.Command("ssh-keygen", + "-N", "", + "-t", "ed25519", + "-f", privKeyPath) + genKey.Stdout = testlogger.New(t) + genKey.Stderr = testlogger.New(t) + if err := genKey.Run(); err != nil { + t.Fatalf("%v: %v", genKey.Args, err) + } + + t.Run("Anon", func(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + _, source, dest := createSourceFiles(t) + + // start a server to sync from + srv := rsynctest.New(t, + rsynctest.InteropModule(source), + rsynctest.Listeners([]rsyncdconfig.Listener{ + { + AnonSSH: "localhost:0", + HostKeyPath: filepath.Join(t.TempDir(), "gokr-rsyncd", "ssh_host_ed25519_key"), + }, + })) + + // sync into dest dir + rsync := exec.Command(rsyncBin, + append( + append([]string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "-e", "ssh -o IdentityFile=" + privKeyPath + " -o StrictHostKeyChecking=no -o CheckHostIP=no -o UserKnownHostsFile=/dev/null -p " + srv.Port, + }, sourcesArgs(t)...), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + // Ensure SSH_* environment variables (like SSH_ASKPASS or + // SSH_AUTH_SOCK) do not leak into the test, otherwise tests + // might be interrupted by a text UI password prompt. + rsync.Env = []string{} // non-nil, but empty + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := sourceFullySyncedTo(t, dest); err != nil { + t.Fatal(err) + } + }) + + t.Run("AuthorizedFail", func(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp, source, dest := createSourceFiles(t) + + authorizedKeysPath := filepath.Join(tmp, "authorized_keys") + if err := os.WriteFile(authorizedKeysPath, []byte("# no keys authorized"), 0644); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, + rsynctest.InteropModule(source), + rsynctest.Listeners([]rsyncdconfig.Listener{ + { + AuthorizedSSH: rsyncdconfig.SSHListener{ + Address: "localhost:0", + AuthorizedKeys: authorizedKeysPath, + }, + }, + })) + + // sync into dest dir + rsync := exec.Command(rsyncBin, + append( + append([]string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "-e", "ssh -o IdentityFile=" + privKeyPath + " -o StrictHostKeyChecking=no -o CheckHostIP=no -o UserKnownHostsFile=/dev/null -p " + srv.Port, + }, sourcesArgs(t)...), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err == nil { + t.Fatalf("rsync unexpectedly succeeded") + } + }) + + t.Run("AuthorizedPass", func(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp, source, dest := createSourceFiles(t) + + authorizedKeysPath := filepath.Join(tmp, "authorized_keys") + pubKey, err := os.ReadFile(privKeyPath + ".pub") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(authorizedKeysPath, pubKey, 0644); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, + rsynctest.InteropModule(source), + rsynctest.Listeners([]rsyncdconfig.Listener{ + { + AuthorizedSSH: rsyncdconfig.SSHListener{ + Address: "localhost:0", + AuthorizedKeys: authorizedKeysPath, + }, + }, + })) + + // sync into dest dir + rsync := exec.Command(rsyncBin, + append( + append([]string{ + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "-e", "ssh -o IdentityFile=" + privKeyPath + " -o StrictHostKeyChecking=no -o CheckHostIP=no -o UserKnownHostsFile=/dev/null -p " + srv.Port, + }, sourcesArgs(t)...), + filepath.Base(dest))...) + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := sourceFullySyncedTo(t, dest); err != nil { + t.Fatal(err) + } + }) +} diff --git a/integration/ipacl/ipacl_test.go b/integration/ipacl/ipacl_test.go new file mode 100644 index 0000000..7b2a153 --- /dev/null +++ b/integration/ipacl/ipacl_test.go @@ -0,0 +1,150 @@ +package ipacl_test + +import ( + "bytes" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" +) + +type connWithRemoteAddrListener struct { + net.Listener + + remoteAddr net.Addr +} + +func (l *connWithRemoteAddrListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + return &connWithRemoteAddr{ + Conn: conn, + remoteAddr: l.remoteAddr, + }, nil +} + +type connWithRemoteAddr struct { + net.Conn + + remoteAddr net.Addr +} + +func (c *connWithRemoteAddr) RemoteAddr() net.Addr { + return c.remoteAddr +} + +func TestIPACL(t *testing.T) { + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + // create files in source to be copied + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + + cfg, err := rsyncdconfig.FromString(` +[[module]] +name = "interop" +path = '` + source + `' +acl = [ + "allow 192.168.1.0/24", + "allow 2001:db8::1/32", + "deny all" +] + +`) + if err != nil { + t.Fatal(err) + } + + for _, tt := range []struct { + remoteAddr string + wantError bool + }{ + { + remoteAddr: "192.168.1.1", + wantError: false, + }, + { + remoteAddr: "2001:db8::1234", + wantError: false, + }, + { + remoteAddr: "10.0.0.1", + wantError: true, + }, + } { + t.Run(tt.remoteAddr, func(t *testing.T) { + t.Parallel() + + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + ln = &connWithRemoteAddrListener{ + Listener: ln, + remoteAddr: &net.TCPAddr{ + IP: net.ParseIP(tt.remoteAddr), + Port: 1234, + }, + } + srv := rsynctest.New(t, cfg.Modules, rsynctest.Listener(ln)) + + // request module list: this should work regardless of the source IP + var buf bytes.Buffer + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "rsync://localhost") + rsync.Stdout = &buf + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + output := buf.String() + if want := "interop\tinterop"; !strings.Contains(output, want) { + t.Fatalf("rsync output unexpectedly did not contain %q:\n%s", want, output) + } + + buf.Reset() + // actually transfer the interop module (in dry-run mode) to trigger + // the ACL check dry run (slight differences in protocol) + rsync = exec.Command(rsynctest.AnyRsync(t), + // "--debug=all4", + "--archive", + "-v", "-v", "-v", "-v", + "--port="+srv.Port, + "--dry-run", + "rsync://localhost/interop/", // copy contents of interop + //source+"/", // sync from local directory + filepath.Base(dest)) // directly into dest + rsync.Dir = filepath.Dir(dest) + rsync.Stdout = testlogger.New(t) + rsync.Stderr = &buf + if err := rsync.Run(); err != nil { + if !tt.wantError { + t.Fatalf("%v: %v", rsync.Args, err) + } + } + + if tt.wantError { + if !strings.Contains(buf.String(), "@ERROR: access denied") { + t.Fatalf("expected access denied error, got: %q", buf.String()) + } + } + }) + } +} diff --git a/integration/receiver/receiver_daemon_test.go b/integration/receiver/receiver_daemon_test.go new file mode 100644 index 0000000..6e8f43f --- /dev/null +++ b/integration/receiver/receiver_daemon_test.go @@ -0,0 +1,301 @@ +package receiver_test + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" +) + +func TestDaemonReceiverSync(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server which receives data + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + // A verbosity level of 3 is enough, any higher than that and rsync + // will start listing individual chunk matches. + "-v", "-v", "-v", // "-v", + "--port="+srv.Port, + filepath.Base(source)+"/", // copy contents of source (relative to rsync.Dir) + "rsync://localhost/interop/") + rsync.Dir = filepath.Dir(source) + rsync.Env = append(os.Environ(), + // Ensure rsync does not localize decimal separators and fractional + // points based on the current locale: + "LANG=C.UTF-8") + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } +} + +// like TestDaemonReceiverSync, but specifying a destination path that is a subdirectory +// within the module. +func TestDaemonReceiverSyncSubdir(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tests := []struct { + name string + destPath string + rsyncTarget string + }{ + { + name: "simple subdir", + destPath: "destsubdir", + rsyncTarget: "rsync://localhost/interop/destsubdir/", + }, + { + name: "nested subdir", + destPath: "subdir/destsubdir", + rsyncTarget: "rsync://localhost/interop/subdir/destsubdir/", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, tt.destPath, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server which receives data + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + // A verbosity level of 3 is enough, any higher than that and rsync + // will start listing individual chunk matches. + "-v", "-v", "-v", // "-v", + "--port="+srv.Port, + filepath.Base(source)+"/", // copy contents of source (relative to rsync.Dir) + tt.rsyncTarget) + rsync.Dir = filepath.Dir(source) + rsync.Env = append(os.Environ(), + // Ensure rsync does not localize decimal separators and fractional + // points based on the current locale: + "LANG=C.UTF-8") + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + }) + } +} + +// like TestDaemonReceiverSync, but specifying a destination path that is a subdirectory +// within the module. +func TestDaemonReceiverSyncSubdirTraversal(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tests := []struct { + name string + destPath string + rsyncTarget string + }{ + { + name: "simple subdir", + destPath: "destsubdir", + rsyncTarget: "rsync://localhost/interop/../", + }, + { + name: "nested subdir", + destPath: "subdir/destsubdir", + rsyncTarget: "rsync://localhost/interop/subdir/../../", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, tt.destPath, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server which receives data + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + // A verbosity level of 3 is enough, any higher than that and rsync + // will start listing individual chunk matches. + "-v", "-v", "-v", // "-v", + "--port="+srv.Port, + filepath.Base(source)+"/", // copy contents of source (relative to rsync.Dir) + tt.rsyncTarget) + rsync.Dir = filepath.Dir(source) + rsync.Env = append(os.Environ(), + // Ensure rsync does not localize decimal separators and fractional + // points based on the current locale: + "LANG=C.UTF-8") + var buf bytes.Buffer + rsync.Stdout = &buf + rsync.Stderr = &buf + if err := rsync.Run(); err != nil { + if strings.Contains(buf.String(), "path escapes from parent") { + return + } + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestDaemonReceiverDelete(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server which receives data + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + run := func() { + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + "--delete", + // A verbosity level of 3 is enough, any higher than that and rsync + // will start listing individual chunk matches. + "-v", "-v", "-v", // "-v", + "--port="+srv.Port, + filepath.Base(source)+"/", // copy contents of source (relative to rsync.Dir) + "rsync://localhost/interop/") + rsync.Dir = filepath.Dir(source) + rsync.Env = append(os.Environ(), + // Ensure rsync does not localize decimal separators and fractional + // points based on the current locale: + "LANG=C.UTF-8") + rsync.Stdout = testlogger.New(t) + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + } + run() + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + + // Add more files to the destination, which should be deleted: + extra := filepath.Join(dest, "extrafile") + if err := os.WriteFile(extra, []byte("deleteme"), 0644); err != nil { + t.Fatal(err) + } + run() + if _, err := os.Stat(extra); !os.IsNotExist(err) { + t.Errorf("expected %s to be deleted, but it still exists", extra) + } +} + +func TestDaemonReceiverSyncHardLinks(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server which receives data + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + "--hard-links", + // A verbosity level of 3 is enough, any higher than that and rsync + // will start listing individual chunk matches. + "-v", "-v", "-v", // "-v", + "--port="+srv.Port, + filepath.Base(source)+"/", // copy contents of source (relative to rsync.Dir) + "rsync://localhost/interop/") + rsync.Dir = filepath.Dir(source) + rsync.Env = append(os.Environ(), + // Ensure rsync does not localize decimal separators and fractional + // points based on the current locale: + "LANG=C.UTF-8") + var buf bytes.Buffer + rsync.Stdout = &buf + rsync.Stderr = &buf + if err := rsync.Run(); err != nil { + if strings.Contains(buf.String(), "gokr-rsync [receiver]: support for hard links not yet implemented") { + return + } + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } +} diff --git a/integration/receiver/receiver_listing_test.go b/integration/receiver/receiver_listing_test.go new file mode 100644 index 0000000..de91ede --- /dev/null +++ b/integration/receiver/receiver_listing_test.go @@ -0,0 +1,71 @@ +package receiver_test + +import ( + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsynctest" +) + +func init() { + // Run this test in UTC so that the printed timestamp matches our expected + // output. + time.Local = time.UTC +} + +func TestReceiverListing(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + // This explicit chmod might seem redundant at first, + // but is required to make the test pass with umask 027. + if err := os.Chmod(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + // This explicit chmod might seem redundant at first, + // but is required to make the test pass with umask 027. + if err := os.Chmod(hello, 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + stdout, _ := rsynctest.Output(t, "gokr-rsync", + "-aH", + "rsync://localhost:"+srv.Port+"/interop/") + want := `drwxr-xr-x 4096 2009/11/10 23:00:00 . +-rw-r--r-- 5 2009/11/10 23:00:00 hello +` + if runtime.GOOS == "windows" { + want = `drwxrwxrwx 4096 2009/11/10 23:00:00 . +-rw-rw-rw- 5 2009/11/10 23:00:00 hello +` + } + if diff := cmp.Diff(want, string(stdout)); diff != "" { + t.Fatalf("unexpected listing: diff (-want +got):\n%s", diff) + } +} diff --git a/integration/receiver/receiver_resume_test.go b/integration/receiver/receiver_resume_test.go new file mode 100644 index 0000000..76dccfa --- /dev/null +++ b/integration/receiver/receiver_resume_test.go @@ -0,0 +1,298 @@ +package receiver_test + +import ( + "bytes" + "context" + "io" + "math/rand" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" + "github.com/runpod/rsync/rsyncd" +) + +// TestDaemonReceiverResumesFromPartial seeds a ".partial" and checks that +// a --partial push reuses it as the delta basis, then commits and drops it. +func TestDaemonReceiverResumesFromPartial(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("Windows CI uses Cygwin rsync, which mangles native Windows source paths in these daemon push tests") + } + + rsyncBin := rsynctest.TridgeOrGTFO(t, "resume needs a real rsync client to push with --partial") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + for _, d := range []string{source, dest} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + + // Incompressible content so the tail can only arrive as literal data. + const fileSize = 4 << 20 + content := make([]byte, fileSize) + rand.New(rand.NewSource(3)).Read(content) + if err := os.WriteFile(filepath.Join(source, "big.bin"), content, 0o644); err != nil { + t.Fatal(err) + } + + final := filepath.Join(dest, "big.bin") + partial := final + ".partial" + // Seed a partial with a correct ~40% prefix, as an interrupt would leave. + if err := os.WriteFile(partial, content[:fileSize*2/5], 0o644); err != nil { + t.Fatal(err) + } + + port, daemonLogs := startResumeDaemon(t, dest) + + push := exec.Command(rsyncBin, + "--archive", + "--partial", // negotiates keep_partial=1 so the receiver resumes + "--port="+port, + source+"/", + "rsync://localhost/interop/") + push.Env = append(os.Environ(), "LANG=C.UTF-8") + push.Stdout = testlogger.New(t) + push.Stderr = testlogger.New(t) + if err := push.Run(); err != nil { + t.Fatalf("%v: %v", push.Args, err) + } + + got, err := os.ReadFile(final) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(content, got) { + t.Fatalf("reconstructed file != source (%d vs %d bytes)", len(got), len(content)) + } + // A successful commit consumes the partial sidecar. + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Errorf("expected %s to be removed after commit, stat err = %v", partial, err) + } + // The daemon only logs a resume when it used the partial as the basis. + waitForLog(t, daemonLogs, "resuming big.bin from partial", 5*time.Second) +} + +// TestDaemonReceiverResumesAfterInterrupt severs a push mid-transfer so the +// daemon retains ".partial" itself, then a second push resumes against it. +func TestDaemonReceiverResumesAfterInterrupt(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("Windows CI uses Cygwin rsync, which mangles native Windows source paths in these daemon push tests") + } + + rsyncBin := rsynctest.TridgeOrGTFO(t, "resume-after-interrupt needs a real rsync client") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + for _, d := range []string{source, dest} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + + // Large and incompressible so the sever leaves a meaningful prefix. + const fileSize = 8 << 20 + content := make([]byte, fileSize) + rand.New(rand.NewSource(1)).Read(content) + if err := os.WriteFile(filepath.Join(source, "big.bin"), content, 0o644); err != nil { + t.Fatal(err) + } + + port, daemonLogs := startResumeDaemon(t, dest) + + partial := filepath.Join(dest, "big.bin.partial") + final := filepath.Join(dest, "big.bin") + + // Sever the first connection after ~1 MiB; later ones pass through. + proxyPort := startSeveringProxy(t, net.JoinHostPort("127.0.0.1", port), 1<<20) + + // Attempt 1: the push is cut mid-stream, so it fails. + attempt1 := exec.Command(rsyncBin, + "--archive", "--partial", + "--port="+proxyPort, + source+"/", "rsync://localhost/interop/") + attempt1.Env = append(os.Environ(), "LANG=C.UTF-8") + attempt1.Stdout = testlogger.New(t) + attempt1.Stderr = testlogger.New(t) + if err := attempt1.Run(); err == nil { + t.Fatal("attempt 1 unexpectedly succeeded; proxy should have severed it") + } + + // The interrupt must retain .partial and leave no final file. + waitForExists(t, partial, 10*time.Second) + if got := statSize(t, partial); got == 0 { + t.Fatal("retained partial is empty") + } + if _, err := os.Stat(final); !os.IsNotExist(err) { + t.Fatalf("final file must not exist after an interrupt, stat err = %v", err) + } + + // Attempt 2: resume through the now-transparent proxy. + attempt2 := exec.Command(rsyncBin, + "--archive", "--partial", + "--port="+proxyPort, + source+"/", "rsync://localhost/interop/") + attempt2.Env = append(os.Environ(), "LANG=C.UTF-8") + attempt2.Stdout = testlogger.New(t) + attempt2.Stderr = testlogger.New(t) + if err := attempt2.Run(); err != nil { + t.Fatalf("resume push failed: %v", err) + } + + got, err := os.ReadFile(final) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(content, got) { + t.Fatalf("resumed file does not match source (got %d bytes, want %d)", len(got), len(content)) + } + if _, err := os.Stat(partial); !os.IsNotExist(err) { + t.Errorf("partial should be removed after a successful commit, stat err = %v", err) + } + // The resume log proves attempt 2 deltaed against attempt 1's partial. + waitForLog(t, daemonLogs, "resuming big.bin from partial", 5*time.Second) +} + +// startResumeDaemon runs a writable daemon (module "interop") and returns its +// port plus an accessor for its captured log output. +func startResumeDaemon(t *testing.T, dest string) (port string, logs func() string) { + t.Helper() + buf := &syncBuffer{} + srv, err := rsyncd.NewServer( + []rsyncd.Module{{Name: "interop", Path: dest, Writable: true}}, + rsyncd.WithStderr(nopWriteCloser{io.MultiWriter(buf, testlogger.New(t))}), + rsyncd.DontRestrict(), + ) + if err != nil { + t.Fatal(err) + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + go func() { _ = srv.Serve(ctx, ln) }() + + _, port, err = net.SplitHostPort(ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + return port, buf.String +} + +// nopWriteCloser adapts an io.Writer to the io.WriteCloser WithStderr wants. +type nopWriteCloser struct{ io.Writer } + +func (nopWriteCloser) Close() error { return nil } + +// syncBuffer is a concurrency-safe buffer: the daemon writes logs from its own +// goroutine while the test reads them. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// startSeveringProxy relays TCP to backend but drops the first connection after +// severAfter bytes client->backend; later connections pass through untouched. +func startSeveringProxy(t *testing.T, backend string, severAfter int64) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + + var conns atomic.Int32 + go func() { + for { + client, err := ln.Accept() + if err != nil { + return + } + sever := conns.Add(1) == 1 + go func() { + defer client.Close() + up, err := net.Dial("tcp", backend) + if err != nil { + return + } + defer up.Close() + go io.Copy(client, up) // backend -> client + if sever { + _, _ = io.CopyN(up, client, severAfter) + return // closes both conns, dropping the transfer + } + _, _ = io.Copy(up, client) // client -> backend + }() + } + }() + + _, port, err := net.SplitHostPort(ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + return port +} + +func waitForExists(t *testing.T, path string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); err == nil { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s to exist", path) +} + +func waitForLog(t *testing.T, logs func() string, substr string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if strings.Contains(logs(), substr) { + return + } + time.Sleep(25 * time.Millisecond) + } + t.Fatalf("daemon never logged %q; logs so far:\n%s", substr, logs()) +} + +func statSize(t *testing.T, path string) int64 { + t.Helper() + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return st.Size() +} diff --git a/integration/receiver/receiver_sshkey_test.go b/integration/receiver/receiver_sshkey_test.go new file mode 100644 index 0000000..30e6796 --- /dev/null +++ b/integration/receiver/receiver_sshkey_test.go @@ -0,0 +1,40 @@ +package receiver_test + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" +) + +func genPrivateKey() (*rsa.PrivateKey, error) { + const bits = 2048 + priv, err := rsa.GenerateKey(rand.Reader, bits) + if err != nil { + return nil, err + } + if err := priv.Validate(); err != nil { + return nil, err + } + return priv, nil +} + +func asPEM(priv *rsa.PrivateKey) []byte { + return pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }) +} + +func genKey(privKeyPath string) error { + priv, err := genPrivateKey() + if err != nil { + return err + } + if err := os.WriteFile(privKeyPath, asPEM(priv), 0600); err != nil { + return err + } + return nil +} diff --git a/integration/receiver/receiver_test.go b/integration/receiver/receiver_test.go new file mode 100644 index 0000000..76910b2 --- /dev/null +++ b/integration/receiver/receiver_test.go @@ -0,0 +1,591 @@ +package receiver_test + +import ( + "io/fs" + "log" + "os" + "os/user" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/rsyncd" +) + +func TestMain(m *testing.M) { + if err := rsynctest.CommandMain(m); err != nil { + log.Fatal(err) + } +} + +func setUid(t *testing.T, fn string) (uid, gid int, verify bool) { + if os.Getuid() != 0 { + return 0, 0, false + } + + u, err := user.Lookup("nobody") + if err != nil { + t.Fatal(err) + } + + uid64, err := strconv.ParseInt(u.Uid, 0, 64) + if err != nil { + t.Fatal(err) + } + uid = int(uid64) + + gid64, err := strconv.ParseInt(u.Gid, 0, 64) + if err != nil { + t.Fatal(err) + } + gid = int(gid64) + + if err := os.Chown(fn, uid, gid); err != nil { + t.Fatal(err) + } + + return uid, gid, true +} + +func TestReceiver(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + sourcesub := filepath.Join(source, "subdir") + if err := os.MkdirAll(sourcesub, 0755); err != nil { + t.Fatal(err) + } + sourcesubempty := filepath.Join(source, "subdirempty") + if err := os.MkdirAll(sourcesubempty, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + hellosub := filepath.Join(sourcesub, "hello") + if err := os.WriteFile(hellosub, []byte("space"), 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hellosub, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(sourcesubempty, mtime, mtime); err != nil { + t.Fatal(err) + } + + if err := os.Symlink("hello", filepath.Join(source, "hey")); err != nil { + t.Fatal(err) + } + + no := filepath.Join(source, "no") + if err := os.WriteFile(no, []byte("no"), 0666); err != nil { + t.Fatal(err) + } + uid, gid, verifyUid := setUid(t, no) + + devices := filepath.Join(source, "devices") + if os.Getuid() == 0 { + rsynctest.CreateDummyDeviceFiles(t, devices) + } + + // start a server to sync from + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "interop", + Path: source, + }) + args := []string{"-aH"} + firstStats := srv.RunClient(t, args, "./", []string{dest}) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } + if verifyUid { + st, err := os.Stat(filepath.Join(dest, "no")) + if err != nil { + t.Fatal(err) + } + gotUID, gotGID := rsynctest.StatUidGid(t, st) + if got, want := gotUID, uid; got != want { + t.Errorf("unexpected uid: got %d, want %d", got, want) + } + if got, want := gotGID, gid; got != want { + t.Errorf("unexpected gid: got %d, want %d", got, want) + } + } + { + st, err := os.Stat(filepath.Join(dest, "subdir", "hello")) + if err != nil { + t.Fatal(err) + } + if got, want := st.ModTime(), mtime; !got.Equal(want) { + t.Errorf("dest/subdir/hello has unexpected mod time: got %v, want %v", got, want) + } + } + { + st, err := os.Stat(filepath.Join(dest, "subdirempty")) + if err != nil { + t.Fatal(err) + } + if got, want := st.ModTime(), mtime; !got.Equal(want) { + t.Errorf("dest/subdirempty has unexpected mod time: got %v, want %v", got, want) + } + } + if os.Getuid() == 0 { + rsynctest.VerifyDummyDeviceFiles(t, devices, filepath.Join(dest, "devices")) + } + + incrementalStats := srv.RunClient(t, args, "./", []string{dest}) + if incrementalStats.Written >= firstStats.Written { + t.Fatalf("incremental run unexpectedly not more efficient than first run: incremental wrote %d bytes, first wrote %d bytes", incrementalStats.Written, firstStats.Written) + } + + // Make a change that is invisible with our current settings: + // change the file contents without changing size and mtime. + if err := os.WriteFile(hello, []byte("moon!"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + // Replace the dest symlink to see if it will be restored + rsynctest.ReplaceSymlink(t, "wrong", filepath.Join(dest, "hey")) + + srv.RunClient(t, args, "./", []string{dest}) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } +} + +func TestReceiverSync(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server to sync from + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "interop", + Path: source, + }) + args := []string{"-aH"} + firstStats := srv.RunClient(t, args, "./", []string{dest}) + t.Logf("firstStats: %+v", firstStats) + // receiver_test.go:211: firstStats: &{Read:91 Written:3146087 Size:3149824} + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + + // Change the middle of the large data file: + bodyPattern = []byte{0x66} + // modify the large data file + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + incrementalStats := srv.RunClient(t, args, "./", []string{dest}) + t.Logf("incrementalStats: %+v", incrementalStats) + if got, want := incrementalStats.Written, int64(2*1024*1024); got >= want { + t.Fatalf("rsync unexpectedly transferred more data than needed: got %d, want < %d", got, want) + } +} + +func TestReceiverSyncDelete(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server to sync from + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "interop", + Path: source, + }) + args := []string{"-aH", "--delete"} + firstStats := srv.RunClient(t, args, "./", []string{dest}) + t.Logf("firstStats: %+v", firstStats) + // receiver_test.go:211: firstStats: &{Read:91 Written:3146087 Size:3149824} + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + + // Add more files to the destination, which should be deleted: + extra := filepath.Join(dest, "extrafile") + if err := os.WriteFile(extra, []byte("deleteme"), 0644); err != nil { + t.Fatal(err) + } + extraDir := filepath.Join(dest, "extradir") + if err := os.MkdirAll(extraDir, 0755); err != nil { + t.Fatal(err) + } + extra2 := filepath.Join(extraDir, "blocker") + if err := os.WriteFile(extra2, []byte("deleteme"), 0644); err != nil { + t.Fatal(err) + } + + srv.RunClient(t, args, "./", []string{dest}) + for _, gone := range []string{extra, extraDir, extra2} { + if _, err := os.Stat(gone); !os.IsNotExist(err) { + t.Errorf("expected %s to be deleted, but it still exists", gone) + } + } +} + +func TestReceiverAlwaysChecksum(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello.txt") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "interop", + Path: source, + }) + args := []string{"-aH"} + srv.RunClient(t, args, "./", []string{dest}) + + desthello := filepath.Join(dest, "hello.txt") + b, err := os.ReadFile(desthello) + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(b)), "world"; got != want { + t.Fatalf("hello.txt: unexpected contents: got %q, want %q", got, want) + } + + // Make a change that is invisible with our current settings: + // change the file contents without changing size and mtime. + if err := os.WriteFile(hello, []byte("moon!"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + args = append(args, "--checksum") + srv.RunClient(t, args, "./", []string{dest}) + + b, err = os.ReadFile(desthello) + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(b)), "moon!"; got != want { + t.Fatalf("hello.txt: unexpected contents: got %q, want %q", got, want) + } +} + +func TestReceiverReadOnlyDir(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello.txt") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chmod(source, 0o555); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.NewInMemory(t, rsyncd.Module{ + Name: "interop", + Path: source, + }) + args := []string{"-aH"} + srv.RunClient(t, args, "./", []string{dest}) + + desthello := filepath.Join(dest, "hello.txt") + b, err := os.ReadFile(desthello) + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(b)), "world"; got != want { + t.Fatalf("hello.txt: unexpected contents: got %q, want %q", got, want) + } + st, err := os.Stat(dest) + if err != nil { + t.Fatal(err) + } + if got, want := st.Mode()&os.ModePerm, fs.FileMode(0o555); got != want { + t.Fatalf("dest: unexpected permission: got %v, want %v", got, want) + } + + // Make a change that is invisible with our current settings: + // change the file contents without changing size and mtime. + if err := os.WriteFile(hello, []byte("moon!"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + args = append(args, "--checksum") + srv.RunClient(t, args, "./", []string{dest}) + + b, err = os.ReadFile(desthello) + if err != nil { + t.Fatal(err) + } + if got, want := strings.TrimSpace(string(b)), "moon!"; got != want { + t.Fatalf("hello.txt: unexpected contents: got %q, want %q", got, want) + } + + st, err = os.Stat(dest) + if err != nil { + t.Fatal(err) + } + if got, want := st.Mode()&os.ModePerm, fs.FileMode(0o555); got != want { + t.Fatalf("dest: unexpected permission: got %v, want %v", got, want) + } + + // Restore write permission so that t.TempDir() cleanup succeeds + if err := os.Chmod(dest, 0755); err != nil { + t.Fatal(err) + } + if err := os.Chmod(source, 0755); err != nil { + t.Fatal(err) + } +} + +func TestReceiverSSH(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("ssh transport test relies on a POSIX ssh client and /dev/null") + } + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "hello"), []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, + rsynctest.InteropModule(source), + rsynctest.Listeners([]rsyncdconfig.Listener{ + { + AnonSSH: "localhost:0", + HostKeyPath: filepath.Join(tmp, "ssh_host_ed25519_key"), + }, + })) + + // ensure the user running the tests (root when doing the privileged run!) + // has an SSH private key: + privKeyPath := filepath.Join(tmp, "ssh_private_key") + if err := genKey(privKeyPath); err != nil { + t.Fatal(err) + } + + // Ensure SSH_* environment variables (like SSH_ASKPASS or + // SSH_AUTH_SOCK) do not leak into the test, otherwise tests + // might be interrupted by a text UI password prompt. + t.Setenv("SSH_ASKPASS", "") + t.Setenv("SSH_AUTH_SOCK", "") + // sync into dest dir + rsynctest.Run(t, "gokr-rsync", + "-aH", + "--dry-run", + "-e", "ssh -vv -o IdentityFile="+privKeyPath+" -o StrictHostKeyChecking=no -o CheckHostIP=no -o UserKnownHostsFile=/dev/null -p "+srv.Port, + "rsync://localhost/interop/", + dest) +} + +func TestReceiverCommand(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "hello"), []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + // sync into dest dir + rsynctest.Run(t, "gokr-rsync", + "-aH", + "--dry-run", + "-e", `"`+os.Args[0]+`"`, + "localhost:"+source+"/", + dest) +} + +// TestReceiverSymlinkTraversal passes by default but is useful to simulate +// a symlink race TOCTOU attack by modifying rsyncd/rsyncd.go. +func TestReceiverSymlinkTraversal(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + if err := os.WriteFile(filepath.Join(tmp, "passwd"), []byte("secret"), 0644); err != nil { + t.Fatal(err) + } + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "passwd") + if err := os.WriteFile(hello, []byte("benign"), 0644); err != nil { + t.Fatal(err) + } + subdir := filepath.Join(source, "dir") + if err := os.MkdirAll(subdir, 0755); err != nil { + t.Fatal(err) + } + subhello := filepath.Join(subdir, "passwd") + if err := os.WriteFile(subhello, []byte("benign"), 0644); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + rsynctest.Run(t, "gokr-rsync", + "-aH", + "rsync://localhost:"+srv.Port+"/interop/", + dest) + + { + want := []byte("benign") + got, err := os.ReadFile(filepath.Join(dest, "passwd")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + + { + want := []byte("benign") + got, err := os.ReadFile(filepath.Join(dest, "dir", "passwd")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} diff --git a/integration/sender/sender_test.go b/integration/sender/sender_test.go new file mode 100644 index 0000000..6833e6a --- /dev/null +++ b/integration/sender/sender_test.go @@ -0,0 +1,804 @@ +package sender_test + +import ( + "bytes" + "log" + "os" + "os/exec" + "os/user" + "path/filepath" + "runtime" + "strconv" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" +) + +func TestMain(m *testing.M) { + if err := rsynctest.CommandMain(m); err != nil { + log.Fatal(err) + } +} + +func setUid(t *testing.T, fn string) (uid, gid int, verify bool) { + if os.Getuid() != 0 { + return 0, 0, false + } + + u, err := user.Lookup("nobody") + if err != nil { + t.Fatal(err) + } + + uid64, err := strconv.ParseInt(u.Uid, 0, 64) + if err != nil { + t.Fatal(err) + } + uid = int(uid64) + + gid64, err := strconv.ParseInt(u.Gid, 0, 64) + if err != nil { + t.Fatal(err) + } + gid = int(gid64) + + if err := os.Chown(fn, uid, gid); err != nil { + t.Fatal(err) + } + + return uid, gid, true +} + +func TestSender(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + if err := os.Symlink("hello", filepath.Join(source, "hey")); err != nil { + t.Fatal(err) + } + + no := filepath.Join(source, "no") + if err := os.WriteFile(no, []byte("no"), 0666); err != nil { + t.Fatal(err) + } + uid, gid, verifyUid := setUid(t, no) + + devices := filepath.Join(source, "devices") + if os.Getuid() == 0 { + rsynctest.CreateDummyDeviceFiles(t, devices) + } + + // start a server to sync to + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + args := []string{ + "gokr-rsync", + "-aH", + source + "/", + "rsync://localhost:" + srv.Port + "/interop/", + } + firstStats := rsynctest.Run(t, args...) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } + if verifyUid { + st, err := os.Stat(filepath.Join(dest, "no")) + if err != nil { + t.Fatal(err) + } + gotUID, gotGID := rsynctest.StatUidGid(t, st) + if got, want := gotUID, uid; got != want { + t.Errorf("unexpected uid: got %d, want %d", got, want) + } + if got, want := gotGID, gid; got != want { + t.Errorf("unexpected gid: got %d, want %d", got, want) + } + } + if os.Getuid() == 0 { + rsynctest.VerifyDummyDeviceFiles(t, devices, filepath.Join(dest, "devices")) + } + + incrementalStats := rsynctest.Run(t, args...) + if incrementalStats.Written >= firstStats.Written { + t.Fatalf("incremental run unexpectedly not more efficient than first run: incremental wrote %d bytes, first wrote %d bytes", incrementalStats.Written, firstStats.Written) + } + + // Make a change that is invisible with our current settings: + // change the file contents without changing size and mtime. + if err := os.WriteFile(hello, []byte("moon!"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + // Replace the dest symlink to see if it will be restored + rsynctest.ReplaceSymlink(t, "wrong", filepath.Join(dest, "hey")) + + rsynctest.Run(t, args...) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } +} + +// like TestSender, but without a trailing slash, i.e. do not copy directory +// contents, but the directory itself. +func TestSenderNoSlash(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + if err := os.Symlink("hello", filepath.Join(source, "hey")); err != nil { + t.Fatal(err) + } + + no := filepath.Join(source, "no") + if err := os.WriteFile(no, []byte("no"), 0666); err != nil { + t.Fatal(err) + } + uid, gid, verifyUid := setUid(t, no) + + devices := filepath.Join(source, "devices") + if os.Getuid() == 0 { + rsynctest.CreateDummyDeviceFiles(t, devices) + } + + // start a server to sync to + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + args := []string{ + "gokr-rsync", + "-aH", + source, + "rsync://localhost:" + srv.Port + "/interop/", + } + firstStats := rsynctest.Run(t, args...) + + dest = filepath.Join(dest, "source") + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } + if verifyUid { + st, err := os.Stat(filepath.Join(dest, "no")) + if err != nil { + t.Fatal(err) + } + gotUID, gotGID := rsynctest.StatUidGid(t, st) + if got, want := gotUID, uid; got != want { + t.Errorf("unexpected uid: got %d, want %d", got, want) + } + if got, want := gotGID, gid; got != want { + t.Errorf("unexpected gid: got %d, want %d", got, want) + } + } + if os.Getuid() == 0 { + rsynctest.VerifyDummyDeviceFiles(t, devices, filepath.Join(dest, "devices")) + } + + incrementalStats := rsynctest.Run(t, args...) + if incrementalStats.Written >= firstStats.Written { + t.Fatalf("incremental run unexpectedly not more efficient than first run: incremental wrote %d bytes, first wrote %d bytes", incrementalStats.Written, firstStats.Written) + } + + // Make a change that is invisible with our current settings: + // change the file contents without changing size and mtime. + if err := os.WriteFile(hello, []byte("moon!"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + // Replace the dest symlink to see if it will be restored + rsynctest.ReplaceSymlink(t, "wrong", filepath.Join(dest, "hey")) + + rsynctest.Run(t, args...) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } +} + +// like TestSender, but using a relative path +func TestSenderRelative(t *testing.T) { + tmp := t.TempDir() + t.Chdir(tmp) + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + // start a server to sync to + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + args := []string{ + "gokr-rsync", + "-aH", + "source", + "rsync://localhost:" + srv.Port + "/interop/", + } + rsynctest.Run(t, args...) + + dest = filepath.Join(dest, "source") + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} + +// TestSenderNamedSourceParentUnreadable is a regression test for +// https://github.com/runpod/rsync/issues/66: syncing a named directory (no +// trailing slash) must not require access to the source’s parent directory. +// +// The sender allowlists the source itself for Landlock, but not its parent, so +// opening the parent (as the pre-fix code did to obtain the source basename) +// is denied. We reproduce that portably — without Landlock — by removing read +// access from the parent while keeping search (execute) access, which likewise +// makes os.OpenRoot(parent) fail but os.OpenRoot(source) succeed. +func TestSenderNamedSourceParentUnreadable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions do not gate reads on Windows") + } + if os.Getuid() == 0 { + t.Skip("root bypasses directory permission checks, so the parent would still be readable") + } + t.Parallel() + + tmp := t.TempDir() + parent := filepath.Join(tmp, "parent") + source := filepath.Join(parent, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "hello"), []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + // Remove read (but keep search) on the parent, then restore it before the + // t.TempDir() cleanup so RemoveAll can recurse back in. + if err := os.Chmod(parent, 0o111); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(parent, 0o755) }) + + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + // Named directory: no trailing slash, so the source directory itself is + // copied and the receiver creates dest/source/hello. + // + // --gokr.dont_restrict: the unreadable parent (not Landlock) provides + // the constraint under test; skipping the Landlock ruleset stays within + // the kernel's limit of 16 stacked rulesets per process (all tests in + // this package share one process). + rsynctest.Run(t, "gokr-rsync", "-aH", "--gokr.dont_restrict", source, "rsync://localhost:"+srv.Port+"/interop/") + + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "source", "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } +} + +// TestSenderNamedFileParentUnreadable is the single-file variant of +// TestSenderNamedSourceParentUnreadable: syncing a named file must not +// require access to the file's parent directory either (issue #66; the +// client allowlists the file itself as a read-only Landlock rule). +func TestSenderNamedFileParentUnreadable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("POSIX directory permissions do not gate reads on Windows") + } + if os.Getuid() == 0 { + t.Skip("root bypasses directory permission checks, so the parent would still be readable") + } + t.Parallel() + + tmp := t.TempDir() + parent := filepath.Join(tmp, "parent") + hello := filepath.Join(parent, "hello") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(parent, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + // Remove read (but keep search) on the parent, then restore it before the + // t.TempDir() cleanup so RemoveAll can recurse back in. + if err := os.Chmod(parent, 0o111); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.Chmod(parent, 0o755) }) + + srv := rsynctest.New(t, rsynctest.WritableInteropModule(dest)) + + // --gokr.dont_restrict: see TestSenderNamedSourceParentUnreadable. + rsynctest.Run(t, "gokr-rsync", "-aH", "--gokr.dont_restrict", hello, "rsync://localhost:"+srv.Port+"/interop/") + + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } +} + +func TestSenderTraversal(t *testing.T) { + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(source, "hello.txt"), []byte("hi"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tmp, "passwd"), []byte("secret"), 0644); err != nil { + t.Fatal(err) + } + + if err := os.MkdirAll(dest, 0755); err != nil { + t.Fatal(err) + } + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + args := []string{ + "gokr-rsync", + "-aH", + "rsync://localhost:" + srv.Port + "/interop/../", + dest + "/", + } + rsynctest.Run(t, args...) + + passwd := filepath.Join(dest, "passwd") + + got, err := os.ReadFile(passwd) + if err == nil { + t.Fatalf("unexpectedly synced /etc/passwd: %s", string(got)) + } +} + +// like TestSender, but both source and dest are local directories +func TestSenderBothLocal(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + if err := os.Symlink("hello", filepath.Join(source, "hey")); err != nil { + t.Fatal(err) + } + + no := filepath.Join(source, "no") + if err := os.WriteFile(no, []byte("no"), 0666); err != nil { + t.Fatal(err) + } + uid, gid, verifyUid := setUid(t, no) + + devices := filepath.Join(source, "devices") + if os.Getuid() == 0 { + rsynctest.CreateDummyDeviceFiles(t, devices) + } + + args := []string{ + "gokr-rsync", + "-aH", + source, + dest, + } + firstStats := rsynctest.Run(t, args...) + + dest = filepath.Join(dest, "source") + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } + if verifyUid { + st, err := os.Stat(filepath.Join(dest, "no")) + if err != nil { + t.Fatal(err) + } + gotUID, gotGID := rsynctest.StatUidGid(t, st) + if got, want := gotUID, uid; got != want { + t.Errorf("unexpected uid: got %d, want %d", got, want) + } + if got, want := gotGID, gid; got != want { + t.Errorf("unexpected gid: got %d, want %d", got, want) + } + } + if os.Getuid() == 0 { + rsynctest.VerifyDummyDeviceFiles(t, devices, filepath.Join(dest, "devices")) + } + + incrementalStats := rsynctest.Run(t, args...) + if incrementalStats.Written >= firstStats.Written { + t.Fatalf("incremental run unexpectedly not more efficient than first run: incremental wrote %d bytes, first wrote %d bytes", incrementalStats.Written, firstStats.Written) + } + + // Make a change that is invisible with our current settings: + // change the file contents without changing size and mtime. + if err := os.WriteFile(hello, []byte("moon!"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(hello, mtime, mtime); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + // Replace the dest symlink to see if it will be restored + rsynctest.ReplaceSymlink(t, "wrong", filepath.Join(dest, "hey")) + + rsynctest.Run(t, args...) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } + { + got, err := os.Readlink(filepath.Join(dest, "hey")) + if err != nil { + t.Fatal(err) + } + want := "hello" + if got != want { + t.Fatalf("unexpected link target: got %q, want %q", got, want) + } + } +} + +// like TestSender, but the source is a single regular file, not a directory +func TestSenderBothLocalFile(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.WriteFile(source, []byte("hey"), 0644); err != nil { + t.Fatal(err) + } + mtime, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + t.Fatal(err) + } + if err := os.Chtimes(source, mtime, mtime); err != nil { + t.Fatal(err) + } + + args := []string{ + "gokr-rsync", + "-avH", + source, + dest, + } + rsynctest.Run(t, args...) + + { + want := []byte("hey") + got, err := os.ReadFile(filepath.Join(dest, "source")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} + +// like TestSenderBothLocalFile, but with an invocation that once caused a hang +// (see issue #43). +func TestSenderBothLocalHang(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + + hello := filepath.Join(source, "hello.txt") + + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + + args := []string{ + "gokr-rsync", + "-rv", + source, + dest, + } + rsynctest.Run(t, args...) + + { + want := []byte("world") + got, err := os.ReadFile(filepath.Join(dest, "source", "hello.txt")) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} + +// like TestSenderBothLocalFile, but with an invocation that used to fail +// with error message "file has changed mid-transfer" (issue #53). +func TestSenderPartial257K(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + want := make([]byte, 1024*257+1) + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + + hello := filepath.Join(source, "hello.txt") + + if err := os.WriteFile(hello, want, 0644); err != nil { + t.Fatal(err) + } + + destFile := filepath.Join(dest, "source", "hello.txt") + if err := os.MkdirAll(filepath.Dir(destFile), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(destFile, []byte{0}, 0644); err != nil { + t.Fatal(err) + } + + args := []string{ + "gokr-rsync", + "-rv", + source, + dest, + } + rsynctest.Run(t, args...) + + { + got, err := os.ReadFile(destFile) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(want, got); diff != "" { + t.Fatalf("unexpected file contents: diff (-want +got):\n%s", diff) + } + } +} + +func TestReceiverCommandDryRun(t *testing.T) { + t.Parallel() + + if runtime.GOOS == "windows" { + t.Skip("stdin not supported on Windows") + } + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + + if err := os.MkdirAll(source, 0755); err != nil { + t.Fatal(err) + } + hello := filepath.Join(source, "hello") + if err := os.WriteFile(hello, []byte("world"), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(dest, 0755); err != nil { + t.Fatal(err) + } + hello = filepath.Join(dest, "hello") + if err := os.WriteFile(hello, []byte("moon"), 0644); err != nil { + t.Fatal(err) + } + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + rsync := exec.Command(rsynctest.AnyRsync(t), + "--dry-run", + "-e", `"`+exe+`"`, + "-a", + filepath.Base(source)+"/", + "localhost:"+dest+"/") + rsync.Dir = filepath.Dir(source) + rsync.Stdout = &buf + rsync.Stderr = testlogger.New(t) + t.Logf("%v", rsync.Args) + if err := rsync.Run(); err != nil { + t.Fatalf("rsync error, output:\n%s", buf.String()) + } +} diff --git a/integration/sync/sync_test.go b/integration/sync/sync_test.go new file mode 100644 index 0000000..3ea64bf --- /dev/null +++ b/integration/sync/sync_test.go @@ -0,0 +1,164 @@ +package sync_test + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" +) + +func TestMain(m *testing.M) { + rsynctest.CommandMain(m) +} + +var statsTransferRe = regexp.MustCompile(`^sent ([0-9,]+) bytes received ([0-9,]+) bytes ([0-9,.]+) bytes/sec$`) + +func TestSyncExtended(t *testing.T) { + t.Parallel() + + rsyncBin := rsynctest.TridgeOrGTFO(t, "TODO: reason") + + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + dest := filepath.Join(tmp, "dest") + destLarge := filepath.Join(dest, "large-data-file") + + headPattern := []byte{0x11} + bodyPattern := []byte{0xbb} + endPattern := []byte{0xee} + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // start a server to sync from + srv := rsynctest.New(t, rsynctest.InteropModule(source)) + + sync := func() int64 { + rsync := exec.Command(rsyncBin, + // "--debug=all4", + "--archive", + // A verbosity level of 3 is enough, any higher than that and rsync + // will start listing individual chunk matches. + "-v", "-v", "-v", // "-v", + "--checksum", + "--port="+srv.Port, + "rsync://localhost/interop/", // copy contents of source + filepath.Base(dest)) + rsync.Dir = filepath.Dir(dest) + rsync.Env = append(os.Environ(), + // Ensure rsync does not localize decimal separators and fractional + // points based on the current locale: + "LANG=C.UTF-8") + var buf bytes.Buffer + rsync.Stdout = &buf + rsync.Stderr = testlogger.New(t) + if err := rsync.Run(); err != nil { + t.Fatalf("%v: %v", rsync.Args, err) + } + + if err := rsynctest.DataFileMatches(destLarge, headPattern, bodyPattern, endPattern); err != nil { + t.Fatal(err) + } + + totalRead := int64(-1) + for _, line := range strings.Split(buf.String(), "\n") { + // log.Printf("rsync output line: %q", line) + if strings.HasPrefix(line, "sent ") { + // e.g.: + // sent 1,590 bytes received 18 bytes 3,216.00 bytes/sec + // total size is 1,188,046 speedup is 738.83 + matches := statsTransferRe.FindStringSubmatch(line) + if len(matches) == 0 { + t.Fatalf("could not parse rsync 'sent' line") + } + + var err error + totalRead, err = strconv.ParseInt(strings.ReplaceAll(matches[2], ",", ""), 0, 64) + if err != nil { + t.Fatal(err) + } + } + } + return totalRead + } + + { + // initial sync into dest dir + totalRead := sync() + if got, want := totalRead, int64(3*1024); got < want { + t.Fatalf("rsync unexpectedly did not read the whole file over the network: got %d, want >= %d", got, want) + } + } + + { + // second sync (unmodified) into dest dir + totalRead := sync() + if got, want := totalRead, int64(512*1024); got >= want { + t.Fatalf("rsync unexpectedly transferred more data than needed: got %d, want < %d", got, want) + } + } + + // Change the middle of the large data file: + bodyPattern = []byte{0x66} + { + // modify the large data file + rsynctest.WriteLargeDataFile(t, source, headPattern, bodyPattern, endPattern) + + // sync modifications into dest dir + totalRead := sync() + + // TODO: verify speedup value, compare to rsync and openrsync + + if got, want := totalRead, int64(2*1024*1024); got >= want { + t.Fatalf("rsync unexpectedly transferred more data than needed: got %d, want < %d", got, want) + } + } +} + +func TestSyncMultipleSources(t *testing.T) { + tmp := t.TempDir() + src1 := filepath.Join(tmp, "src1") + src2 := filepath.Join(tmp, "src2") + dest := filepath.Join(tmp, "dest") + if err := os.MkdirAll(src1, 0755); err != nil { + t.Fatal(err) + } + const hello = "world" + if err := os.WriteFile(filepath.Join(src1, "hello"), []byte(hello), 0644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(src2, 0755); err != nil { + t.Fatal(err) + } + const bye = "moon" + if err := os.WriteFile(filepath.Join(src2, "bye"), []byte(bye), 0644); err != nil { + t.Fatal(err) + } + + rsynctest.Run(t, "gokr-rsync", + "-av", + src1, + src2, + dest) + + hellob, err := os.ReadFile(filepath.Join(dest, "src1", "hello")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(hellob, []byte(hello)) { + t.Errorf("file 'hello' has unexpected contents: got %q, want %q", string(hellob), hello) + } + + byeb, err := os.ReadFile(filepath.Join(dest, "src2", "bye")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(byeb, []byte(bye)) { + t.Errorf("file 'bye' has unexpected contents: got %q, want %q", string(byeb), bye) + } +} diff --git a/internal/anonssh/anonssh.go b/internal/anonssh/anonssh.go new file mode 100644 index 0000000..509c654 --- /dev/null +++ b/internal/anonssh/anonssh.go @@ -0,0 +1,341 @@ +package anonssh + +import ( + "bufio" + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "encoding/binary" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "os" + "path/filepath" + "strings" + + "github.com/google/shlex" + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/internal/rsyncos" + "golang.org/x/crypto/ssh" +) + +type mainFunc func(args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error + +type anonssh struct { + cfg *rsyncdconfig.Config + main mainFunc + osenv *rsyncos.Env +} + +// env is a Environment Variable request as per RFC4254 6.4. +type env struct { + VariableName string + VariableValue string +} + +// execR is a Command request as per RFC4254 6.5. +type execR struct { + Command string +} + +type session struct { + channel ssh.Channel + anonssh *anonssh +} + +func (s *session) request(ctx context.Context, req *ssh.Request) error { + _ = ctx // FIXME: not yet used + + switch req.Type { + + case "env": + var r env + if err := ssh.Unmarshal(req.Payload, &r); err != nil { + return err + } + + s.anonssh.osenv.Logf("env request: %s=%s", r.VariableName, r.VariableValue) + //s.env = append(s.env, fmt.Sprintf("%s=%s", r.VariableName, r.VariableValue)) + + case "exec": + var r execR + if err := ssh.Unmarshal(req.Payload, &r); err != nil { + return err + } + + cmdline, err := shlex.Split(r.Command) + if err != nil { + return err + } + + s.anonssh.osenv.Logf("cmdline: %q", cmdline) + // 2021/09/12 21:25:34 cmdline: ["rsync" "--server" "--daemon" "."] + go func() { + stderr := s.channel.Stderr() + err := s.anonssh.main(cmdline, s.channel, s.channel, struct { + io.Writer + io.Closer + }{ + Writer: stderr, + Closer: s.channel, + }) + if err != nil { + fmt.Fprintf(stderr, "%s\n", err) + } + + status := make([]byte, 4) + if err != nil { + binary.BigEndian.PutUint32(status, 1) + } + + // See https://tools.ietf.org/html/rfc4254#section-6.10 + if _, err := s.channel.SendRequest("exit-status", false /* wantReply */, status); err != nil { + s.anonssh.osenv.Logf("err2: %v", err) + } + s.channel.Close() + }() + + if req.WantReply { + req.Reply(true, nil) + } + + return nil + + default: + return fmt.Errorf("unknown request type: %q", req.Type) + } + + return nil +} + +func (as *anonssh) handleSession(newChannel ssh.NewChannel) { + channel, requests, err := newChannel.Accept() + if err != nil { + as.osenv.Logf("Could not accept channel (%s)", err) + return + } + + // Sessions have out-of-band requests such as "shell", "pty-req" and "env" + go func(channel ssh.Channel, requests <-chan *ssh.Request) { + ctx, canc := context.WithCancel(context.Background()) + defer canc() + s := session{ + channel: channel, + anonssh: as, + } + for req := range requests { + if err := s.request(ctx, req); err != nil { + s.anonssh.osenv.Logf("request(%q): %v", req.Type, err) + errmsg := []byte(err.Error()) + // Append a trailing newline; the error message is + // displayed as-is by ssh(1). + if errmsg[len(errmsg)-1] != '\n' { + errmsg = append(errmsg, '\n') + } + req.Reply(false, errmsg) + channel.Write(errmsg) + channel.Close() + } + } + s.anonssh.osenv.Logf("SSH requests exhausted") + }(channel, requests) +} + +func (as *anonssh) handleChannel(newChan ssh.NewChannel) { + switch t := newChan.ChannelType(); t { + case "session": + as.handleSession(newChan) + default: + newChan.Reject(ssh.UnknownChannelType, fmt.Sprintf("unknown channel type: %q", t)) + return + } +} + +func genHostKey(keyPath string) ([]byte, error) { + _, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + + x509b, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + return nil, err + } + privateKeyPEM := &pem.Block{ + Type: "PRIVATE KEY", + Bytes: x509b, + } + f, err := os.OpenFile(keyPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return nil, err + } + defer f.Close() + + b := pem.EncodeToMemory(privateKeyPEM) + if _, err := f.Write(b); err != nil { + return nil, err + } + if err := f.Close(); err != nil { + return nil, err + } + + return b, nil +} + +type Listener struct { + hostKey ssh.Signer + authorizedKeys map[string]bool + authorizedKeysPath string +} + +func ListenerFromConfig(osenv *rsyncos.Env, cfg rsyncdconfig.Listener) (*Listener, error) { + hostKeyPath := cfg.HostKeyPath + if hostKeyPath == "" { + dir, err := os.UserConfigDir() + if err != nil { + return nil, err + } + hostKeyPath = filepath.Join(dir, "gokr-rsyncd", "ssh_host_ed25519_key") + } + hostKey, err := loadHostKey(hostKeyPath) + if err != nil { + return nil, err + } + + var authorizedKeys map[string]bool + if cfg.AuthorizedSSH.Address != "" { + if cfg.AuthorizedSSH.AuthorizedKeys == "" { + return nil, fmt.Errorf("authorized_keys not specified") + } + + var err error + authorizedKeys, err = loadAuthorizedKeys(osenv, cfg.AuthorizedSSH.AuthorizedKeys) + if err != nil { + return nil, err + } + } + + return &Listener{ + hostKey: hostKey, + authorizedKeys: authorizedKeys, + authorizedKeysPath: cfg.AuthorizedSSH.AuthorizedKeys, + }, nil +} + +func loadHostKey(path string) (ssh.Signer, error) { + b, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return nil, err + } + b, err = genHostKey(path) + if err != nil { + return nil, err + } + // fall-through + } else { + return nil, err + } + } + return ssh.ParsePrivateKey(b) +} + +func loadAuthorizedKeys(osenv *rsyncos.Env, path string) (map[string]bool, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + result := make(map[string]bool) + + s := bufio.NewScanner(bytes.NewReader(b)) + for lineNum := 1; s.Scan(); lineNum++ { + if tr := strings.TrimSpace(s.Text()); tr == "" || strings.HasPrefix(tr, "#") { + continue + } + pubKey, _, _, _, err := ssh.ParseAuthorizedKey(s.Bytes()) + + // This warning can be removed once the mentioned issue is resolved + if keyType := pubKey.Type(); keyType == "ssh-rsa" { + osenv.Logf("Warning: ignoring unsupported ssh-rsa key in %s:%d (see https://github.com/gokrazy/breakglass/issues/11)", path, lineNum) + } + + if err != nil { + return nil, err + } + result[string(pubKey.Marshal())] = true + } + if err := s.Err(); err != nil { + return nil, err + } + + return result, nil +} + +func Serve(ctx context.Context, osenv *rsyncos.Env, ln net.Listener, listener *Listener, cfg *rsyncdconfig.Config, main mainFunc) error { + go func() { + <-ctx.Done() + ln.Close() // unblocks Accept() + }() + + as := &anonssh{ + cfg: cfg, + main: main, + osenv: osenv, + } + + config := &ssh.ServerConfig{ + PublicKeyCallback: func(conn ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { + if listener.authorizedKeys == nil { + osenv.Logf("user %q successfully authorized from remote addr %s", conn.User(), conn.RemoteAddr()) + return nil, nil + } + if listener.authorizedKeys[string(pubKey.Marshal())] { + osenv.Logf("user %q successfully authorized from remote addr %s", conn.User(), conn.RemoteAddr()) + return nil, nil + } + return nil, fmt.Errorf("public key not found in %s", listener.authorizedKeysPath) + }, + } + + config.AddHostKey(listener.hostKey) + + osenv.Logf("SSH host key fingerprint: %s", ssh.FingerprintSHA256(listener.hostKey.PublicKey())) + + for { + conn, err := ln.Accept() + if err != nil { + select { + case <-ctx.Done(): + return nil // ignore expected 'use of closed network connection' error on context cancel + default: + } + + if errors.Is(err, net.ErrClosed) { + return err + } + osenv.Logf("accept: %v", err) + continue + } + + go func(conn net.Conn) { + _, chans, reqs, err := ssh.NewServerConn(conn, config) + if err != nil { + osenv.Logf("handshake: %v", err) + return + } + + // discard all out of band requests + go ssh.DiscardRequests(reqs) + + for newChannel := range chans { + as.handleChannel(newChannel) + } + }(conn) + } +} diff --git a/internal/log/logger.go b/internal/log/logger.go new file mode 100644 index 0000000..399a86c --- /dev/null +++ b/internal/log/logger.go @@ -0,0 +1,22 @@ +// Package log defines the logger interface used in rsync library. +package log + +import ( + "io" + "log" +) + +// Logger is an interface that allows specifying your own logger. +// By default, the Go log package is used, which prints to stderr. +type Logger interface { + // Printf logs message to the underlying log output. Arguments are handled in the manner of fmt.Printf. + Printf(msg string, a ...any) + + Output(calldepth int, s string) error +} + +const logFlags = log.LstdFlags | log.Lshortfile + +func New(out io.Writer) Logger { + return log.New(out, "", logFlags) +} diff --git a/internal/log/logger_test.go b/internal/log/logger_test.go new file mode 100644 index 0000000..a3246ec --- /dev/null +++ b/internal/log/logger_test.go @@ -0,0 +1,23 @@ +package log_test + +import ( + "bytes" + "fmt" + + "github.com/runpod/rsync/internal/log" +) + +type fakeLogger struct { + out *bytes.Buffer +} + +var _ log.Logger = (*fakeLogger)(nil) + +func (f *fakeLogger) Printf(msg string, a ...any) { + fmt.Fprintf(f.out, msg, a...) +} + +func (f *fakeLogger) Output(calldepth int, s string) error { + fmt.Fprintf(f.out, "%s", s) + return nil +} diff --git a/internal/maincmd/clientmaincmd.go b/internal/maincmd/clientmaincmd.go new file mode 100644 index 0000000..0fe5e7e --- /dev/null +++ b/internal/maincmd/clientmaincmd.go @@ -0,0 +1,449 @@ +package maincmd + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/google/shlex" + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/progress" + "github.com/runpod/rsync/internal/receiver" + "github.com/runpod/rsync/internal/restrict" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncstats" + "github.com/runpod/rsync/internal/rsyncwire" + "github.com/runpod/rsync/internal/sender" +) + +// rsync/main.c:start_client +func rsyncMain(ctx context.Context, osenv *rsyncos.Env, opts *rsyncopts.Options, sources []string, dest string) (*rsyncstats.TransferStats, error) { + if opts.Verbose() { + osenv.Logf("dest: %q, sources: %q", dest, sources) + osenv.Logf("opts: %+v", opts) + } + // Guaranteed to be non-empty by caller of rsyncMain(). + src := sources[0] + + if opts.Verbose() { + osenv.Logf("processing src=%s", src) + } + daemonConnection := 0 // no daemon + host, path, port, err := checkForHostspec(src) + if opts.Verbose() { + osenv.Logf("host=%q, path=%q, port=%d, err=%v", host, path, port, err) + } + if err != nil { + // source is local, check dest arg + opts.SetSender() + // TODO: remote_argv == "."? + host, path, port, err = checkForHostspec(dest) + if opts.Verbose() { + osenv.Logf("host=%q, path=%q, port=%d, err=%v", host, path, port, err) + } + if path == "" { + if opts.Verbose() { + osenv.Logf("source and dest are both local!") + } + host = "" + port = 0 + path = dest + opts.SetLocalServer() + } else { + // dest is remote + if port != 0 { + if opts.ShellCommand() != "" { + daemonConnection = 1 // daemon via remote shell + } else { + daemonConnection = -1 // daemon via socket + } + } + } + } else { + // source is remote + if port != 0 { + if opts.ShellCommand() != "" { + daemonConnection = 1 // daemon via remote shell + } else { + daemonConnection = -1 // daemon via socket + } + } + } + + // TODO: if opts.AmSender(), verify extra source args have no hostspec + var roDirs, rwDirs []string + other := dest + if other != "" { + if !opts.Sender() || opts.LocalServer() { + // dest is local + if err := os.MkdirAll(other, 0755); err != nil { + return nil, err + } + } + } + paths := []string{other} + if opts.Sender() { + // source is local + // other = src + paths = sources + roDirs = sources + if opts.LocalServer() { + // source and dest are both local + rwDirs = []string{dest} + } + } else { + if other != "" { + rwDirs = paths + } + } + + if daemonConnection < 0 { + stats, err := socketClient(ctx, osenv, opts, host, path, port, paths, roDirs, rwDirs) + if err != nil { + return nil, err + } + return stats, nil + } + + machine := host + user := "" + if idx := strings.IndexByte(machine, '@'); idx > -1 { + user = machine[:idx] + machine = machine[idx+1:] + } + rc, wc, err := doCmd(osenv, opts, machine, user, path, daemonConnection) + if err != nil { + return nil, err + } + defer rc.Close() + defer wc.Close() + conn := &readWriteCloser{ + r: rc, + w: wc, + } + + if osenv.Restrict() { + if err := restrict.MaybeFileSystem(roDirs, rwDirs); err != nil { + return nil, err + } + } + + negotiate := true + if daemonConnection != 0 { + done, err := StartInbandExchange(osenv, opts, conn, path) + if err != nil { + return nil, err + } + if done { + return nil, nil + } + negotiate = false // already done + } + stats, err := ClientRun(osenv, opts, conn, paths, negotiate) + if err != nil { + return nil, err + } + return stats, nil +} + +// rsync/main.c:do_cmd +func doCmd(osenv *rsyncos.Env, opts *rsyncopts.Options, machine, user, path string, daemonConnection int) (io.ReadCloser, io.WriteCloser, error) { + if opts.Verbose() { + osenv.Logf("doCmd(machine=%q, user=%q, path=%q, daemonConnection=%d)", + machine, user, path, daemonConnection) + } + var args []string + if !opts.LocalServer() { + cmd := opts.ShellCommand() + if cmd == "" { + cmd = "ssh" + if e := os.Getenv("RSYNC_RSH"); e != "" { + cmd = e + } + } + + // We use shlex.Split(), whereas rsync implements its own shell-style-like + // parsing. The nuances likely don’t matter to any users, and if so, users + // might prefer shell-style parsing. + // + // On Windows, double the backslashes first: shlex treats "\" as an escape + // character, which would otherwise mangle a native "-e C:\path\rsync.exe" + // into "C:pathrsync.exe". + if runtime.GOOS == "windows" { + cmd = strings.ReplaceAll(cmd, `\`, `\\`) + } + var err error + args, err = shlex.Split(cmd) + if err != nil { + return nil, nil, err + } + + if user != "" && daemonConnection == 0 /* && !dashlset */ { + args = append(args, "-l", user) + } + + args = append(args, machine) + + args = append(args, "rsync") // TODO: flag + } else { + // NOTE: tridge rsync will fork and run child_main(), we call Main() in + // a separate goroutine below. + args = append(args, os.Args[0]) + } + + if daemonConnection > 0 { + args = append(args, "--server", "--daemon") + } else { + args = append(args, opts.ServerOptions()...) + } + args = append(args, ".") + + if daemonConnection == 0 { + args = append(args, path) + } + + if opts.Verbose() { + osenv.Logf("args: %q", args) + } + + if opts.LocalServer() { + stdinrd, stdinwr := io.Pipe() + stdoutrd, stdoutwr := io.Pipe() + go func() { + defer stdinrd.Close() + defer stdoutwr.Close() + osenv := &rsyncos.Env{ + Stdin: stdinrd, + Stdout: stdoutwr, + Stderr: os.Stderr, + // If this transfer restricts file system access, all goroutines + // (including the other end of the connection) are affected. + DontRestrict: true, + } + _, err := Main(context.Background(), osenv, args, nil) + if err != nil { + osenv.Logf("Main(): %v", err) + } + }() + return stdoutrd, stdinwr, nil + } + + ssh := exec.Command(args[0], args[1:]...) + wc, err := ssh.StdinPipe() + if err != nil { + return nil, nil, err + } + rc, err := ssh.StdoutPipe() + if err != nil { + return nil, nil, err + } + ssh.Stderr = osenv.Stderr + if err := ssh.Start(); err != nil { + return nil, nil, err + } + + go func() { + // TODO: correctly terminate the main process when the underlying SSH + // process exits. + if err := ssh.Wait(); err != nil { + osenv.Logf("remote shell exited: %v", err) + } + }() + + return rc, wc, nil +} + +// rsync/main.c:client_run +func ClientRun(osenv *rsyncos.Env, opts *rsyncopts.Options, conn io.ReadWriteCloser, paths []string, negotiate bool) (*rsyncstats.TransferStats, error) { + crd := &rsyncwire.CountingReader{R: conn} + cwr := &rsyncwire.CountingWriter{W: conn} + c := &rsyncwire.Conn{ + Reader: crd, + Writer: cwr, + } + + if negotiate { + if err := c.WriteInt32(rsync.ProtocolVersion); err != nil { + return nil, err + } + remoteProtocol, err := c.ReadInt32() + if err != nil { + return nil, err + } + if opts.Verbose() { + osenv.Logf("remote protocol: %d", remoteProtocol) + } + } + + seed, err := c.ReadInt32() + if err != nil { + return nil, fmt.Errorf("reading seed: %v", err) + } + + mrd := &rsyncwire.MultiplexReader{ + Env: osenv, + Reader: conn, + } + // TODO: rearchitect such that our buffer can be smaller than the largest + // rsync message size + rd := bufio.NewReaderSize(mrd, 256*1024) + // Update crd to track the multiplexed reader, + // but copy the number of bytes read. + crd = &rsyncwire.CountingReader{ + R: struct { + io.Reader + io.Closer + }{ + Reader: rd, + Closer: conn, + }, + BytesRead: crd.BytesRead, + } + c.Reader = crd + + if opts.Sender() { + st := &sender.Transfer{ + Logger: osenv.Logger(), + Opts: opts, + Conn: c, + Seed: seed, + Env: osenv, + Progress: progress.NewPrinter(osenv.Stdout, time.Now), + } + if opts.Verbose() { + osenv.Logf("sender(paths=%q)", paths) + } + + // Turn relative paths like ./gcexportdata or bin/gcexportdata + // into absolute paths so that we can call Transfer.Do() + // with modPath="/" below. + for idx, path := range paths { + // Trailing slashes are meaningful to rsync, + // so preserve a trailing slash across filepath.Abs. + hasTrailingSlash := strings.HasSuffix(path, "/") + abs, err := filepath.Abs(path) + if err != nil { + return nil, err + } + paths[idx] = abs + if hasTrailingSlash { + paths[idx] += "/" + } + } + + stats, err := st.Do(crd, cwr, rsync.FileSystemRoot, paths, nil) + if err != nil { + return nil, err + } + return stats, nil + } + + if len(paths) != 1 { + return nil, fmt.Errorf("BUG: expected exactly one path, got %q", paths) + } + + rt := &receiver.Transfer{ + Logger: osenv.Logger(), + Opts: &receiver.TransferOpts{ + Verbose: opts.Verbose(), + DryRun: opts.DryRun(), + Progress: opts.Progress(), + + DeleteMode: opts.DeleteMode(), + PreserveGid: opts.PreserveGid(), + PreserveUid: opts.PreserveUid(), + PreserveLinks: opts.PreserveLinks(), + PreservePerms: opts.PreservePerms(), + PreserveDevices: opts.PreserveDevices(), + PreserveSpecials: opts.PreserveSpecials(), + PreserveTimes: opts.PreserveMTimes(), + PreserveHardlinks: opts.PreserveHardLinks(), + IgnoreTimes: opts.IgnoreTimes(), + AlwaysChecksum: opts.AlwaysChecksum(), + DoFsync: opts.DoFsync(), + + InfoGTE: opts.InfoGTE, + DebugGTE: opts.DebugGTE, + + KeepPartial: opts.KeepPartial(), + }, + Dest: paths[0], + Env: osenv, + Conn: c, + Seed: seed, + Progress: progress.NewPrinter(osenv.Stdout, time.Now), + } + if opts.Verbose() { + osenv.Logf("receiving to dest=%s", rt.Dest) + } + if rt.Dest == "" { + // just listing modules, not transferring anything + } else { + if err := os.MkdirAll(rt.Dest, 0755); err != nil { + return nil, fmt.Errorf("MkdirAll(dest=%s): %v", rt.Dest, err) + } + rt.DestRoot, err = os.OpenRoot(rt.Dest) + if err != nil { + return nil, fmt.Errorf("OpenRoot(dest=%s): %v", rt.Dest, err) + } + defer rt.DestRoot.Close() + if osenv.Restrict() { + if err := restrict.MaybeFileSystem(nil, []string{rt.Dest}); err != nil { + return nil, fmt.Errorf("landlock: %v", err) + } + } + } + + for _, rule := range opts.FilterRules() { + c.WriteInt32(int32(len(rule))) + c.WriteString(rule) + } + const exclusionListEnd = 0 + if err := c.WriteInt32(exclusionListEnd); err != nil { + return nil, err + } + + if opts.DebugGTE(rsyncopts.DEBUG_RECV, 1) { + osenv.Logf("exclusion list sent") + } + + // receive file list + if opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + osenv.Logf("receiving file list") + } + fileList, err := rt.ReceiveFileList() + if err != nil { + return nil, err + } + if opts.DebugGTE(rsyncopts.DEBUG_FLIST, 2) { + osenv.Logf("received %d names", len(fileList)) + } + + return rt.Do(c, fileList, false) +} + +func clientMain(ctx context.Context, osenv *rsyncos.Env, opts *rsyncopts.Options, remaining []string) (*rsyncstats.TransferStats, error) { + if len(remaining) == 0 { + // help goes to stderr when no arguments were specified + fmt.Fprintln(osenv.Stderr, opts.Help()) + return nil, fmt.Errorf("rsync error: syntax or usage error") + } + if len(remaining) == 1 { + // Usages with just one SRC arg and no DEST arg list the source files + // instead of copying. + dest := "" + sources := remaining + return rsyncMain(ctx, osenv, opts, sources, dest) + } + dest := remaining[len(remaining)-1] + sources := remaining[:len(remaining)-1] + return rsyncMain(ctx, osenv, opts, sources, dest) +} diff --git a/internal/maincmd/clientserver.go b/internal/maincmd/clientserver.go new file mode 100644 index 0000000..615debc --- /dev/null +++ b/internal/maincmd/clientserver.go @@ -0,0 +1,164 @@ +package maincmd + +import ( + "bufio" + "context" + "fmt" + "io" + "net" + "strconv" + "strings" + "time" + + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/restrict" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncstats" +) + +// rsync/clientserver.c:start_socket_client +func socketClient(ctx context.Context, osenv *rsyncos.Env, opts *rsyncopts.Options, host string, remotePath string, port int, paths []string, roDirs, rwDirs []string) (*rsyncstats.TransferStats, error) { + if port < 0 { + if port := opts.RsyncPort(); port > 0 { + host += ":" + strconv.Itoa(port) + } else { + host += ":873" // rsync daemon port + } + } else { + host += ":" + strconv.Itoa(port) + } + dialer := net.Dialer{ + // Prefer the Go resolver: We know which files it uses (which makes life + // easier for the restrict package), whereas the C resolver can be + // extended by host-specific plugins. + Resolver: &net.Resolver{ + PreferGo: true, + }, + } + timeoutStr := "" + if timeout := opts.ConnectTimeoutSeconds(); timeout > 0 { + dialer.Timeout = time.Duration(timeout) * time.Second + timeoutStr = fmt.Sprintf(" (timeout: %d seconds)", timeout) + } + dialFn := dialer.DialContext + if osenv.DialContext != nil { + dialFn = osenv.DialContext + osenv.Logf("Opening TCP connection to %s%s (via custom DialContext)", host, timeoutStr) + } else { + osenv.Logf("Opening TCP connection to %s%s", host, timeoutStr) + } + conn, err := dialFn(ctx, "tcp", host) + if err != nil { + return nil, err + } + defer conn.Close() + + if osenv.Restrict() { + if err := restrict.MaybeFileSystem(roDirs, rwDirs); err != nil { + return nil, err + } + } + done, err := StartInbandExchange(osenv, opts, conn, remotePath) + if err != nil { + return nil, err + } + if done { + return nil, nil + } + stats, err := ClientRun(osenv, opts, conn, paths, false) + if err != nil { + return nil, err + } + return stats, nil +} + +// rsync/clientserver.c:start_inband_exchange +func StartInbandExchange(osenv *rsyncos.Env, opts *rsyncopts.Options, conn io.ReadWriter, remotePath string) (done bool, _ error) { + module := remotePath + if idx := strings.IndexByte(module, '/'); idx > -1 { + module = module[:idx] + } + osenv.Logf("rsync module %q, path %q", module, remotePath) + + rd := bufio.NewReader(conn) + + // send client greeting + fmt.Fprintf(conn, "@RSYNCD: %d\n", rsync.ProtocolVersion) + + // read server greeting + serverGreeting, err := rd.ReadString('\n') + if err != nil { + return false, fmt.Errorf("ReadString: %v", err) + } + serverGreeting = strings.TrimSpace(serverGreeting) + const serverGreetingPrefix = "@RSYNCD: " + if !strings.HasPrefix(serverGreeting, serverGreetingPrefix) { + return false, fmt.Errorf("invalid server greeting: got %q", serverGreeting) + } + // protocol negotiation: require at least version 27 + serverGreeting = strings.TrimPrefix(serverGreeting, serverGreetingPrefix) + var remoteProtocol, remoteSub int32 + if _, err := fmt.Sscanf(serverGreeting, "%d.%d", &remoteProtocol, &remoteSub); err != nil { + if _, err := fmt.Sscanf(serverGreeting, "%d", &remoteProtocol); err != nil { + return false, fmt.Errorf("reading server greeting: %v", err) + } + } + if remoteProtocol < 27 { + return false, fmt.Errorf("server version %d too old", remoteProtocol) + } + + if opts.Verbose() { + osenv.Logf("(Client) Protocol versions: remote=%d, negotiated=%d", remoteProtocol, rsync.ProtocolVersion) + osenv.Logf("Client checksum: md4") + } + + // send module name + fmt.Fprintf(conn, "%s\n", module) + for { + line, err := rd.ReadString('\n') + if err != nil { + return false, fmt.Errorf("did not get server startup line: %v", err) + } + line = strings.TrimSpace(line) + if opts.DebugGTE(rsyncopts.DEBUG_PROTO, 1) { + osenv.Logf("read line: %q", line) + } + + if strings.HasPrefix(line, "@RSYNCD: AUTHREQD ") { + // TODO: implement support for authentication + return false, fmt.Errorf("authentication not yet implemented") + } + + if line == "@RSYNCD: OK" { + break + } + + if line == "@RSYNCD: EXIT" { + return true, nil + } + + if strings.HasPrefix(line, "@ERROR") { + fmt.Fprintf(osenv.Stderr, "%s\n", line) + return false, fmt.Errorf("abort (rsync fatal error)") + } + + if opts.OutputMOTD() { + // print rsync server message of the day (MOTD) + fmt.Fprintf(osenv.Stdout, "%s\n", line) + } + } + + sargv := opts.ServerOptions() + sargv = append(sargv, ".") + sargv = append(sargv, remotePath) + if opts.Verbose() { + osenv.Logf("sending daemon args: %s", sargv) + } + for _, argv := range sargv { + fmt.Fprintf(conn, "%s\n", argv) + } + fmt.Fprintf(conn, "\n") + + return false, nil +} diff --git a/internal/maincmd/listeners.go b/internal/maincmd/listeners.go new file mode 100644 index 0000000..bedcc51 --- /dev/null +++ b/internal/maincmd/listeners.go @@ -0,0 +1,9 @@ +//go:build !linux + +package maincmd + +import "net" + +func systemdListeners() ([]net.Listener, error) { + return nil, nil +} diff --git a/internal/maincmd/listeners_linux.go b/internal/maincmd/listeners_linux.go new file mode 100644 index 0000000..be535f2 --- /dev/null +++ b/internal/maincmd/listeners_linux.go @@ -0,0 +1,24 @@ +//go:build linux + +package maincmd + +import ( + "fmt" + "net" + + "github.com/coreos/go-systemd/activation" +) + +func systemdListeners() ([]net.Listener, error) { + listeners, err := activation.Listeners() + if err != nil { + return nil, err + } + if len(listeners) == 0 { + return nil, nil + } + if got, want := len(listeners), 1; got != want { + return nil, fmt.Errorf("unexpected number of sockets received from systemd: got %d, want %d", got, want) + } + return listeners, nil +} diff --git a/internal/maincmd/maincmd.go b/internal/maincmd/maincmd.go new file mode 100644 index 0000000..4f9ac68 --- /dev/null +++ b/internal/maincmd/maincmd.go @@ -0,0 +1,316 @@ +// Package maincmd implements a subset of the '$ rsync' CLI surface, namely that it can: +// - serve as a server daemon over TCP or SSH (via SSH session stdin/stdout) +// - act as "client" CLI for connecting to the server +// - Not yet implemented: both "client" and "server" can act as the sender and the receiver +package maincmd + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "os" + "strings" + + "github.com/runpod/rsync/internal/anonssh" + "github.com/runpod/rsync/internal/restrict" + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncstats" + "github.com/runpod/rsync/rsyncd" + + // For profiling and debugging + _ "net/http/pprof" +) + +func version(osenv *rsyncos.Env) { + osenv.Logf("gokrazy rsync, pid %d", os.Getpid()) +} + +type readWriteCloser struct { + r io.Reader + w io.Writer + c io.Closer +} + +func (r *readWriteCloser) Read(p []byte) (n int, err error) { return r.r.Read(p) } +func (r *readWriteCloser) Write(p []byte) (n int, err error) { return r.w.Write(p) } +func (r *readWriteCloser) Close() error { return r.c.Close() } + +func Main(ctx context.Context, osenv *rsyncos.Env, args []string, cfg *rsyncdconfig.Config) (*rsyncstats.TransferStats, error) { + osenv.Logf("Main(osenv=%v, args=%q)", osenv, args) + pc := rsyncopts.NewContext(rsyncopts.NewOptionsWithGokrazyDefaults(osenv)) + if err := pc.ParseArguments(osenv, args[1:]); err != nil { + if pe, ok := err.(*rsyncopts.PoptError); ok && + pe.Errno == rsyncopts.POPT_ERROR_BADOPT && + strings.HasPrefix(pe.Error(), "--gokr.") { + return nil, fmt.Errorf("%v (you need to specify --daemon before flags starting with --gokr are available)", pe) + } + return nil, err + } + opts := pc.Options + remaining := pc.RemainingArgs + // osenv.Logf("remaining: %v", remaining) + + // calling convention: daemon mode over remote shell (also builtin SSH) + // Example: --server --daemon . + if opts.Daemon() && opts.Server() { + // start_daemon() + if cfg == nil { + var err error + cfg, _, err = rsyncdconfig.FromDefaultFiles() + if err != nil { + return nil, err + } + } + rsyncdOpts := []rsyncd.Option{ + rsyncd.WithStderr(osenv.Stderr), + } + if osenv.DontRestrict { + rsyncdOpts = append(rsyncdOpts, rsyncd.DontRestrict()) + } + srv, err := rsyncd.NewServer(cfg.Modules, rsyncdOpts...) + if err != nil { + return nil, err + } + conn := rsyncd.NewConnection(osenv.Stdin, osenv.Stdout, "") + return nil, srv.HandleDaemonConn(ctx, conn) + } + + // calling convention: command mode (over remote shell or locally) + // Example: --server --sender -vvvvlogDtpre.iLsfxCIvu . . + if opts.Server() { + // start_server() + srv, err := rsyncd.NewServer(nil, rsyncd.WithStderr(osenv.Stderr)) + if err != nil { + return nil, err + } + + // TODO: remove duplication with handleDaemonConn + if len(remaining) < 2 { + return nil, fmt.Errorf("invalid args: at least one directory required") + } + if got, want := remaining[0], "."; got != want { + return nil, fmt.Errorf("protocol error: got %q, expected %q", got, want) + } + paths := remaining[1:] + if opts.Verbose() { + osenv.Logf("paths: %q", paths) + } + var roDirs, rwDirs []string + if opts.Sender() { + roDirs = append(roDirs, paths...) + } else { + for _, path := range paths { + if err := os.MkdirAll(path, 0755); err != nil { + return nil, err + } + } + rwDirs = append(rwDirs, paths...) + } + if osenv.Restrict() { + if err := restrict.MaybeFileSystem(roDirs, rwDirs); err != nil { + return nil, err + } + } + conn := rsyncd.NewConnection(osenv.Stdin, osenv.Stdout, "") + return nil, srv.InternalHandleConn(ctx, conn, nil, pc) + } + + if !opts.Daemon() { + if !osenv.DontRestrict { + osenv.DontRestrict = opts.GokrazyClient.DontRestrict == 1 + } + return clientMain(ctx, osenv, opts, remaining) + } + + // daemon_main() + + // calling convention: start a daemon in TCP listening mode (or with systemd + // socket activation) + + var cfgfn string + var cfgErr error + if cfg == nil { + if opts.GokrazyDaemon.Config != "" { + cfgfn = opts.GokrazyDaemon.Config + cfg, cfgErr = rsyncdconfig.FromFile(cfgfn) + } else { + cfg, cfgfn, cfgErr = rsyncdconfig.FromDefaultFiles() + } + if cfgErr != nil { + if os.IsNotExist(cfgErr) { + osenv.Logf("config file not found, relying on flags") + // a non-existant config file is not an error: users can start + // gokr-rsyncd with e.g. the -gokr.listen and -gokr.modulemap flags. + cfg = &rsyncdconfig.Config{ + Listeners: []rsyncdconfig.Listener{ + { + Rsyncd: opts.GokrazyDaemon.Listen, + AnonSSH: opts.GokrazyDaemon.AnonSSHListen, + }, + }, + Modules: []rsyncd.Module{}, + } + } else { + return nil, cfgErr + } + } else { + osenv.Logf("config file %s loaded", cfgfn) + } + } + + if os.IsNotExist(cfgErr) { + if opts.GokrazyDaemon.Listen == "" && + opts.GokrazyDaemon.AnonSSHListen == "" { + return nil, fmt.Errorf("neither -gokr.listen nor -gokr.anonssh_listen specified, and config file not found: %v", cfgErr) + } + // If no config file was found, and the user did not specify a + // -gokr.modulemap flag, use a default value to force the user to + // configure a module map. + if opts.GokrazyDaemon.ModuleMap == "" { + opts.GokrazyDaemon.ModuleMap = "nonex=/nonexistant/path" + } + } else { + if len(cfg.Listeners) == 0 || + (cfg.Listeners[0].Rsyncd == "" && + cfg.Listeners[0].AnonSSH == "" && + cfg.Listeners[0].AuthorizedSSH.Address == "") { + return nil, fmt.Errorf("no rsyncd listeners configured, add a [[listener]] to %s", cfgfn) + } + } + // TODO: loosen this restriction, create multiple listeners + + if len(cfg.Listeners) != 1 || + (cfg.Listeners[0].Rsyncd == "" && + cfg.Listeners[0].AnonSSH == "" && + cfg.Listeners[0].AuthorizedSSH.Address == "") { + return nil, fmt.Errorf("not precisely 1 rsyncd listener specified") + } + + var sshListener *anonssh.Listener + listenAddr := cfg.Listeners[0].Rsyncd + if listenAddr == "" { + listenAddr = cfg.Listeners[0].AnonSSH + if listenAddr == "" { + listenAddr = cfg.Listeners[0].AuthorizedSSH.Address + var err error + sshListener, err = anonssh.ListenerFromConfig(osenv, cfg.Listeners[0]) + if err != nil { + return nil, err + } + } else { + var err error + sshListener, err = anonssh.ListenerFromConfig(osenv, cfg.Listeners[0]) + if err != nil { + return nil, err + } + } + } + + if moduleMap := opts.GokrazyDaemon.ModuleMap; moduleMap != "" { + parts := strings.Split(moduleMap, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("malformed -gokr.modulemap parameter %q, expected =", moduleMap) + } + module := rsyncd.Module{ + Name: parts[0], + Path: parts[1], + } + cfg.Modules = append(cfg.Modules, module) + } + if cfg.DontNamespace { + if cfg.Listeners[0].Rsyncd != "" || + cfg.Listeners[0].AnonSSH != "" { + return nil, fmt.Errorf("dont_namespace must be used with authorized_ssh listeners only") + } + version(osenv) + osenv.Logf("environment: not namespace due to dont_namespace option") + } else { + if err := namespace(osenv, cfg.Modules, listenAddr); err == errIsParent { + return nil, nil + } else if err != nil { + return nil, fmt.Errorf("namespace: %v", err) + } + } + osenv.Logf("%d rsync modules configured in total", len(cfg.Modules)) + for _, mod := range cfg.Modules { + if !cfg.DontNamespace && !mod.Writable { + if err := canUnexpectedlyWriteTo(mod.Path); err != nil { + return nil, err + } + } + + osenv.Logf("rsync module %q with path %s configured", mod.Name, mod.Path) + } + + if monitoringListen := opts.GokrazyDaemon.MonitoringListen; monitoringListen != "" { + go func() { + osenv.Logf("HTTP server for monitoring listening on http://%s/debug/pprof", monitoringListen) + if err := http.ListenAndServe(monitoringListen, nil); err != nil { + osenv.Logf("-monitoring_listen: %v", err) + } + }() + } + + srv, err := rsyncd.NewServer(cfg.Modules, rsyncd.WithStderr(osenv.Stderr)) + if err != nil { + return nil, err + } + var ln net.Listener + listeners, err := systemdListeners() + if err != nil { + return nil, err + } + if len(listeners) > 0 { + ln = listeners[0] + } else { + osenv.Logf("not using systemd socket activation, creating listener") + ln, err = net.Listen("tcp", listenAddr) + if err != nil { + return nil, err + } + } + + if cfg.Listeners[0].AuthorizedSSH.Address != "" { + if cfg.Listeners[0].AuthorizedSSH.AuthorizedKeys == "" { + return nil, fmt.Errorf("misconfiguration: authorized_keys must not be empty when using an authorized_ssh listener") + } + osenv.Logf("rsync daemon listening (authorized SSH) on %s", ln.Addr()) + return nil, anonssh.Serve(ctx, osenv, ln, sshListener, cfg, func(args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { + osenv := &rsyncos.Env{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + // This process is already restricted since to the + // rsyncd.NewServer call above. Do not add more rulesets to stay + // under the limit of policy layers per process. + DontRestrict: true, + } + _, err := Main(ctx, osenv, args, cfg) + return err + }) + } + + if cfg.Listeners[0].AnonSSH != "" { + osenv.Logf("rsync daemon listening (anon SSH) on %s", ln.Addr()) + return nil, anonssh.Serve(ctx, osenv, ln, sshListener, cfg, func(args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { + osenv := &rsyncos.Env{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + // This process is already restricted since to the + // rsyncd.NewServer call above. Do not add more rulesets to stay + // under the limit of policy layers per process. + DontRestrict: true, + } + _, err := Main(ctx, osenv, args, cfg) + return err + }) + } + + osenv.Logf("rsync daemon listening on rsync://%s", ln.Addr()) + return nil, srv.Serve(ctx, ln) +} diff --git a/internal/maincmd/namespacing.go b/internal/maincmd/namespacing.go new file mode 100644 index 0000000..2fecb28 --- /dev/null +++ b/internal/maincmd/namespacing.go @@ -0,0 +1,72 @@ +//go:build !linux || nonamespacing + +package maincmd + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "strconv" + + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/rsyncd" +) + +func namespace(osenv *rsyncos.Env, modules []rsyncd.Module, listen string) error { + if os.Getenv("GOKRAZY_RSYNC_PRIVDROP") != "" { + osenv.Logf("pid %d (privileges dropped)", os.Getpid()) + + // Expected by the go-systemd package, and hard to set before creating + // the process in Go. + os.Setenv("LISTEN_PID", strconv.Itoa(os.Getpid())) + + return nil + } + + if os.Getuid() != 0 { + version(osenv) + osenv.Logf("environment: unprivileged") + return nil + } + + version(osenv) + osenv.Logf("environment: privileged") + osenv.Logf("running as root (uid 0), dropping privileges to nobody (uid/gid 65534)") + + exe, err := os.Executable() + if err != nil { + return err + } + + // Create the listener while still running as uid 0 and inherit it, so that + // we can listen on port 873 (rsync), which requires CAP_NET_BIND_SERVICE. + ln, err := net.Listen("tcp", listen) + if err != nil { + return err + } + + cmd := exec.Command(exe, os.Args[1:]...) + cmd.Dir = "/" + // TODO: clean the environment + cmd.Env = append(os.Environ(), + "GOKRAZY_RSYNC_PRIVDROP=1", + "LISTEN_FDS=1", // ExtraFiles start at 3 + "PATH=/bin:"+os.Getenv("PATH")) + cmd.Stdin = os.Stdin // for interactive debugging + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + lnFile, err := ln.(*net.TCPListener).File() + if err != nil { + return err + } + cmd.ExtraFiles = []*os.File{lnFile} + runAsUnprivilegedUser(cmd) + if err := cmd.Run(); err != nil { + return fmt.Errorf("%v: %v", cmd.Args, err) + } + return errIsParent +} + +var errIsParent = errors.New("re-exec parent process sentinel error") diff --git a/internal/maincmd/namespacing_linux.go b/internal/maincmd/namespacing_linux.go new file mode 100644 index 0000000..1d6f944 --- /dev/null +++ b/internal/maincmd/namespacing_linux.go @@ -0,0 +1,188 @@ +//go:build linux && !nonamespacing + +package maincmd + +import ( + "errors" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" + + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/rsyncd" + "golang.org/x/sys/unix" +) + +// based on https://medium.com/@teddyking/namespaces-in-go-mount-e4c04fe9fb29 +func pivotRoot(newroot string) error { + putold := filepath.Join(newroot, "/.pivot_root") + + // create putold directory + if err := os.MkdirAll(putold, 0700); err != nil { + return err + } + + // bind mount newroot to itself - this is a slight hack + // needed to work around a pivot_root requirement + err := syscall.Mount( + newroot, + newroot, + "", + syscall.MS_BIND|syscall.MS_REC, + "") + if err != nil { + return fmt.Errorf("mount(): %v", err) + } + + // remount root as read-only: https://unix.stackexchange.com/questions/128336/why-doesnt-mount-respect-the-read-only-option-for-bind-mounts + err = syscall.Mount( + newroot, + newroot, + "", + syscall.MS_BIND|syscall.MS_REMOUNT|syscall.MS_RDONLY, + "") + if err != nil { + return fmt.Errorf("mount -o remount,ro: %v", err) + } + + // TODO: when trying to use syscall.PivotRoot(".", ".") as described in + // https://manpages.debian.org/bullseye/manpages-dev/pivot_root.2.en.html#NOTES, + // I would end up with writeable rsync module mounts?! + if err := syscall.PivotRoot(newroot, putold); err != nil { + return fmt.Errorf("pivot_root(2): %v", err) + } + + if err := os.Chdir("/"); err != nil { + return fmt.Errorf("chdir(/): %v", err) + } + + // Only unmount the old root, but not delete it: we lack permission now that + // the root is re-mounted read-only. + putold = "/.pivot_root" + if err := syscall.Unmount(putold, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("unmount(%s): %v", putold, err) + } + + return nil +} + +func namespace(osenv *rsyncos.Env, modules []rsyncd.Module, listen string) error { + if os.Getenv("GOKRAZY_RSYNC_NAMESPACE") != "" { + osenv.Logf("pid %d (inside Linux mount/pid namespace)", os.Getpid()) + + // Expected by the go-systemd package, and hard to set before creating + // the process in Go. + os.Setenv("LISTEN_PID", strconv.Itoa(os.Getpid())) + + // Set mount point propagation to MS_SLAVE. + // See https://hechao.li/2020/06/09/Mini-Container-Series-Part-1-Filesystem-Isolation/ + if err := syscall.Mount("", "/", "", syscall.MS_REC|syscall.MS_SLAVE, ""); err != nil { + return fmt.Errorf("mount(/, MS_SLAVE): %v", err) + } + + // Create our own tmpfs mount so that we won’t end up with nodev, + // nosuid, noexec and/or atime, to prevent operation not permitted + // errors when remounting read-only(): + // https://unix.stackexchange.com/questions/655409/in-a-user-namespace-as-non-root-on-a-nosuid-nodev-filesystem-why-does-a-bind-m + tmpdir, err := os.MkdirTemp("", "gokr-rsync") + if err != nil { + return err + } + if err := syscall.Mount("tmpfs", tmpdir, "tmpfs", syscall.MS_REC, ""); err != nil { + return fmt.Errorf("mount(tmpfs, %s): %v", tmpdir, err) + } + + if err := os.Chdir(tmpdir); err != nil { + return err + } + + // prepare read-only bind mounts for each configured rsync module: + osenv.Logf("mounting rsync modules read-only:") + for _, mod := range modules { + osenv.Logf(" rsync module %q from host=%s to namespace=/%s", mod.Name, mod.Path, mod.Name) + // TODO: restrict module names to not contain slashes. does rsync do that? + if err := os.MkdirAll(mod.Name, 0755); err != nil { + return err + } + if err := syscall.Mount(mod.Path, mod.Name, "none", syscall.MS_BIND|syscall.MS_RDONLY, ""); err != nil { + return err + } + } + + wd, err := os.Getwd() + if err != nil { + return err + } + + if err := pivotRoot(wd); err != nil { + return fmt.Errorf("pivotRoot(%q): %v", wd, err) + } + + if err := dropPrivileges(osenv); err != nil { + return fmt.Errorf("dropPrivileges: %v", err) + } + + for idx, mod := range modules { + mod.Path = "/" + mod.Name + modules[idx] = mod + } + + if err := canUnexpectedlyWriteTo("."); err != nil { + return err + } + + return nil + } + + if os.Getuid() != 0 { + version(osenv) + osenv.Logf("environment: unprivileged") + return nil + } + + version(osenv) + osenv.Logf("environment: privileged") + osenv.Logf("creating Linux mount/pid namespace for read-only rsync module mounts") + + exe, err := os.Executable() + if err != nil { + return err + } + + // Create the listener while still running as uid 0 and inherit it, so that + // we can listen on port 873 (rsync), which requires CAP_NET_BIND_SERVICE. + ln, err := net.Listen("tcp", listen) + if err != nil { + return err + } + + cmd := exec.Command(exe, os.Args[1:]...) + cmd.Dir = "/" + // TODO: clean the environment + cmd.Env = append(os.Environ(), + "GOKRAZY_RSYNC_NAMESPACE=1", + "LISTEN_FDS=1", // ExtraFiles start at 3 + "PATH=/bin:"+os.Getenv("PATH")) + cmd.Stdin = os.Stdin // for interactive debugging + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + lnFile, err := ln.(*net.TCPListener).File() + if err != nil { + return err + } + cmd.ExtraFiles = []*os.File{lnFile} + cmd.SysProcAttr = &syscall.SysProcAttr{ + Cloneflags: unix.CLONE_NEWNS | unix.CLONE_NEWPID, + GidMappingsEnableSetgroups: false, + } + if err := cmd.Run(); err != nil { + return fmt.Errorf("%v: %v", cmd.Args, err) + } + return errIsParent +} + +var errIsParent = errors.New("re-exec parent process sentinel error") diff --git a/internal/maincmd/options.go b/internal/maincmd/options.go new file mode 100644 index 0000000..7dc6a0e --- /dev/null +++ b/internal/maincmd/options.go @@ -0,0 +1,129 @@ +package maincmd + +import ( + "fmt" + "os" + "strconv" + "strings" + "unicode" +) + +// parseHostspec returns the [USER@]HOST part of the string +// +// rsync/options.c:parse_hostspec +func parseHostspec(src string, parsingURL bool) (host, path string, port int, _ error) { + var userlen int + var hostlen int + var hoststart int + i := 0 + for ; i <= len(src); i++ { + if i == len(src) { + if !parsingURL { + return "", "", 0, fmt.Errorf("ran out of string") + } + if hostlen == 0 { + hostlen = len(src[hoststart:]) + } + break + } + + s := src[i] + if s == ':' || s == '/' { + if hostlen == 0 { + hostlen = len(src[hoststart:i]) + } + i++ + if s == '/' { + if !parsingURL { + return "", "", 0, fmt.Errorf("/, but not parsing URL") + } + } else if s == ':' && parsingURL { + rest := src[i:] + digits := "" + for _, s := range rest { + if !unicode.IsDigit(s) { + break + } + digits += string(s) + } + if digits != "" { + p, err := strconv.ParseInt(digits, 0, 64) + if err != nil { + return "", "", port, err + } + port = int(p) + i += len(digits) + } + if i < len(src) && src[i] != '/' { + return "", "", 0, fmt.Errorf("expected / or end, got %q", src[i:]) + } + if i < len(src) { + i++ + } + } + break + } + if s == '@' { + userlen = i + 1 + hoststart = i + 1 + } else if s == '[' { + if i != hoststart { + return "", "", 0, fmt.Errorf("brackets not at host position") + } + hoststart++ + for i < len(src) && src[i] != ']' && src[i] != '/' { + i++ + } + hostlen = len(src[hoststart : i+1]) + if i == len(src) || + src[i] != ']' || + (i < len(src)-1 && src[i+1] != '/' && src[i+1] != ':') || + hostlen == 0 { + return "", "", 0, fmt.Errorf("WTF") + } + } + } + if userlen > 0 { + host = src[:userlen] + hostlen += userlen + } + host += src[hoststart:hostlen] + + // On Windows, a local disk path like C:\rsync parses as + // host="C", path="\\rsync". Detect that and error out. + isDriveLetter := len(host) == 1 && + ((host[0] >= 'A' && host[0] <= 'Z') || + (host[0] >= 'a' && host[0] <= 'z')) + if isDriveLetter && src[i] == os.PathSeparator { + return "", "", 0, fmt.Errorf("local disk path detected") + } + + return host, src[i:], port, nil +} + +// rsync/options.c:check_for_hostspec +func checkForHostspec(src string) (host, path string, port int, _ error) { + if strings.HasPrefix(src, "rsync://") { + var err error + if host, path, port, err = parseHostspec(strings.TrimPrefix(src, "rsync://"), true); err == nil { + if port == 0 { + port = -1 + } + return host, path, port, nil + } + } + var err error + host, path, port, err = parseHostspec(src, false) + if err != nil { + return host, path, port, err + } + if strings.HasPrefix(path, ":") { + if port == 0 { + port = -1 + } + path = strings.TrimPrefix(path, ":") + return host, path, port, nil + } + port = 0 // not a daemon-accessing spec + return host, path, port, nil +} diff --git a/internal/maincmd/options_test.go b/internal/maincmd/options_test.go new file mode 100644 index 0000000..26584f5 --- /dev/null +++ b/internal/maincmd/options_test.go @@ -0,0 +1,222 @@ +package maincmd + +import ( + "fmt" + "testing" +) + +func TestParseHostspec(t *testing.T) { + for _, tt := range []struct { + src string + parsingURL bool + wantHost string + wantPath string + wantPort int + }{ + { + src: "localhost", + parsingURL: true, + wantHost: "localhost", + wantPath: "", + wantPort: 0, + }, + + { + src: "localhost/path", + parsingURL: true, + wantHost: "localhost", + wantPath: "path", + wantPort: 0, + }, + + { + src: "user@localhost/path", + parsingURL: true, + wantHost: "user@localhost", + wantPath: "path", + wantPort: 0, + }, + + { + src: "user@[2001:db8::1]:23/path", + parsingURL: true, + wantHost: "user@2001:db8::1", + wantPath: "path", + wantPort: 23, + }, + + { + src: "localhost:881", + parsingURL: true, + wantHost: "localhost", + wantPath: "", + wantPort: 881, + }, + + { + src: "localhost:881/path", + parsingURL: true, + wantHost: "localhost", + wantPath: "path", + wantPort: 881, + }, + + { + src: "localhost:/path", + parsingURL: true, + wantHost: "localhost", + wantPath: "path", + wantPort: 0, + }, + + { + src: "localhost:", + parsingURL: false, + wantHost: "localhost", + wantPath: "", + wantPort: 0, + }, + + { + src: "localhost:path", + parsingURL: false, + wantHost: "localhost", + wantPath: "path", + wantPort: 0, + }, + + { + src: "localhost::path", + parsingURL: false, + wantHost: "localhost", + wantPath: ":path", + wantPort: 0, + }, + + { + src: "user@localhost::path", + parsingURL: false, + wantHost: "user@localhost", + wantPath: ":path", + wantPort: 0, + }, + } { + t.Run(fmt.Sprintf("src=%s, url=%v", tt.src, tt.parsingURL), func(t *testing.T) { + host, path, port, err := parseHostspec(tt.src, tt.parsingURL) + if err != nil { + t.Fatal(err) + } + if host != tt.wantHost { + t.Errorf("unexpected host: got %q, want %q", host, tt.wantHost) + } + if path != tt.wantPath { + t.Errorf("unexpected path: got %q, want %q", path, tt.wantPath) + } + if port != tt.wantPort { + t.Errorf("unexpected port: got %d, want %d", port, tt.wantPort) + } + }) + } +} + +func TestCheckForHostspec(t *testing.T) { + for _, tt := range []struct { + src string + wantHost string + wantPath string + wantPort int + }{ + { + src: "rsync://localhost", + wantHost: "localhost", + wantPath: "", + wantPort: -1, // daemon-accessing + }, + + { + src: "rsync://localhost/path", + wantHost: "localhost", + wantPath: "path", + wantPort: -1, // daemon-accessing + }, + + { + src: "rsync://user@localhost/path", + wantHost: "user@localhost", + wantPath: "path", + wantPort: -1, // daemon-accessing + }, + + { + src: "user@[2001:db8::1]:path", + wantHost: "user@2001:db8::1", + wantPath: "path", + wantPort: 0, // non-daemon-accessing + }, + + { + src: "localhost:path", + wantHost: "localhost", + wantPath: "path", + wantPort: 0, // non-daemon-accessing + }, + + { + src: "user@localhost:path", + wantHost: "user@localhost", + wantPath: "path", + wantPort: 0, // non-daemon-accessing + }, + + { + src: "localhost:/path", + wantHost: "localhost", + wantPath: "/path", + wantPort: 0, // non-daemon-accessing + }, + + { + src: "localhost:", + wantHost: "localhost", + wantPath: "", + wantPort: 0, // non-daemon-accessing + }, + + { + src: "localhost:path", + wantHost: "localhost", + wantPath: "path", + wantPort: 0, // non-daemon-accessing + }, + + { + src: "localhost::path", + wantHost: "localhost", + wantPath: "path", + wantPort: -1, // daemon-accessing + }, + + { + src: "user@localhost::path", + wantHost: "user@localhost", + wantPath: "path", + wantPort: -1, // daemon-accessing + }, + } { + t.Run(tt.src, func(t *testing.T) { + host, path, port, err := checkForHostspec(tt.src) + if err != nil { + t.Fatal(err) + } + if host != tt.wantHost { + t.Errorf("unexpected host: got %q, want %q", host, tt.wantHost) + } + if path != tt.wantPath { + t.Errorf("unexpected path: got %q, want %q", path, tt.wantPath) + } + if port != tt.wantPort { + t.Errorf("unexpected port: got %d, want %d", port, tt.wantPort) + } + }) + } +} diff --git a/internal/maincmd/privdrop.go b/internal/maincmd/privdrop.go new file mode 100644 index 0000000..3455e98 --- /dev/null +++ b/internal/maincmd/privdrop.go @@ -0,0 +1,38 @@ +//go:build linux && !nonamespacing + +package maincmd + +import ( + "fmt" + "syscall" + + "github.com/runpod/rsync/internal/rsyncos" +) + +func dropPrivileges(osenv *rsyncos.Env) error { + if syscall.Getuid() != 0 { + return nil + } + + osenv.Logf("running as root (uid 0), dropping privileges to nobody (uid/gid 65534)") + if err := syscall.Setgid(65534); err != nil { + return fmt.Errorf("setgid(65534): %v", err) + } + + if err := syscall.Setuid(65534); err != nil { + return fmt.Errorf("setuid(65534): %v", err) + } + + // Defense in depth: exit if we can re-gain uid/gid 0 permission: + if err := syscall.Setgid(0); err == nil { + //lint:ignore ST1005 we need this punctuation for dramatic effect! + return fmt.Errorf("unexpectedly able to re-gain gid 0 permission!") + } + + if err := syscall.Setuid(0); err == nil { + //lint:ignore ST1005 we need this punctuation for dramatic effect! + return fmt.Errorf("unexpectedly able to re-gain uid 0 permission!") + } + + return nil +} diff --git a/internal/maincmd/unprivileged.go b/internal/maincmd/unprivileged.go new file mode 100644 index 0000000..378ba1b --- /dev/null +++ b/internal/maincmd/unprivileged.go @@ -0,0 +1,10 @@ +//go:build !linux + +package maincmd + +import ( + "os/exec" +) + +func runAsUnprivilegedUser(*exec.Cmd) { +} diff --git a/internal/maincmd/unprivileged_linux.go b/internal/maincmd/unprivileged_linux.go new file mode 100644 index 0000000..5fabe82 --- /dev/null +++ b/internal/maincmd/unprivileged_linux.go @@ -0,0 +1,17 @@ +//go:build linux && nonamespacing + +package maincmd + +import ( + "os/exec" + "syscall" +) + +func runAsUnprivilegedUser(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{ + Uid: 65534, + Gid: 65534, + }, + } +} diff --git a/internal/maincmd/writetest.go b/internal/maincmd/writetest.go new file mode 100644 index 0000000..aab911f --- /dev/null +++ b/internal/maincmd/writetest.go @@ -0,0 +1,16 @@ +package maincmd + +import ( + "fmt" + "os" + "path/filepath" +) + +func canUnexpectedlyWriteTo(dir string) error { + fn := filepath.Join(dir, "gokr-rsyncd.unexpectedly_writable") + if err := os.WriteFile(fn, []byte("gokr-rsyncd creates this file to prevent misconfigurations. if you see this file, it means gokr-rsyncd unexpectedly was started with too many privileges"), 0644); err == nil { + os.Remove(fn) + return fmt.Errorf("unexpectedly able to write file to %s, exiting", dir) + } + return nil +} diff --git a/internal/progress/progress.go b/internal/progress/progress.go new file mode 100644 index 0000000..7e4be75 --- /dev/null +++ b/internal/progress/progress.go @@ -0,0 +1,119 @@ +package progress + +import ( + "fmt" + "io" + "time" +) + +type progressAt struct { + when time.Time + offset uint64 +} + +type Printer struct { + // config + out io.Writer + now func() time.Time + + // state + first bool + size uint64 + history [5]progressAt + oldest int // index into history +} + +func NewPrinter(out io.Writer, now func() time.Time) Printer { + p := Printer{ + out: out, + now: now, + } + n := now() + for i := range 5 { + p.history[i] = progressAt{ + when: n, + offset: 0, + } + } + return p +} + +func (p *Printer) Reset(size uint64) { + now := p.now() + p.size = size + p.first = true + for i := range 5 { + p.history[i] = progressAt{ + when: now, + offset: 0, + } + } +} + +func (p *Printer) MaybeShow(offset uint64, last bool) { + newest := p.oldest + if newest == 0 { + newest = 4 + } else { + newest-- + } + now := p.now() + if !last && now.Sub(p.history[newest].when) < 1*time.Second { + return + } + p.Show(offset, last) +} + +func (p *Printer) Show(offset uint64, last bool) { + now := p.now() + newest := p.oldest + p.oldest = (p.oldest + 1) % 5 + p.history[newest] = progressAt{ + when: now, + offset: offset, + } + + pct := int(float64(offset) / float64(p.size) * 100) + + oldestOffset := p.history[p.oldest].offset + diff := now.Sub(p.history[p.oldest].when).Seconds() + if diff == 0 { + diff = 1 + } + rate := float64(offset-oldestOffset) / diff + var remainSec float64 // seconds + if rate > 0 { + remainSec = float64(p.size-offset) / rate + } + + rate /= 1024 + unit := "kB/s" + switch { + case rate > 1024*1024: + rate /= 1024 * 1024 + unit = "GB/s" + case rate > 1024: + rate /= 1024 + unit = "MB/s" + } + + remaining := " ??:??:??" + if remainSec >= 0 && remainSec <= 9999*3600 { + remaining = fmt.Sprintf("%4d:%02d:%02d", + int(remainSec/3600), + int(remainSec/60)%60, + int(remainSec)%60) + } + + if p.first { + p.first = false + } else { + p.out.Write([]byte{'\r'}) + } + fmt.Fprintf(p.out, "%15d %3d%% %7.2f%s %s", offset, pct, rate, unit, remaining) + if last { + // TODO: show where we are within the file list + // (number of files transferred vs. number of files total) + p.out.Write([]byte{'\n'}) + } +} diff --git a/internal/progress/progress_test.go b/internal/progress/progress_test.go new file mode 100644 index 0000000..dcc9d02 --- /dev/null +++ b/internal/progress/progress_test.go @@ -0,0 +1,26 @@ +package progress + +import ( + "bytes" + "testing" + "time" +) + +func TestProgress(t *testing.T) { + now := time.Now() + var buf bytes.Buffer + p := NewPrinter(&buf, func() time.Time { + return now + }) + p.Reset(1234) + p.Show(0, false) + if got, want := buf.String(), " 0 0% 0.00kB/s 0:00:00"; got != want { + t.Errorf("progress.Show(0) = %q, want %q", got, want) + } + now = now.Add(1 * time.Second) + buf.Reset() + p.Show(617, false) + if got, want := buf.String(), "\r 617 50% 0.60kB/s 0:00:01"; got != want { + t.Errorf("progress.Show(617) = %q, want %q", got, want) + } +} diff --git a/internal/receiver/do.go b/internal/receiver/do.go new file mode 100644 index 0000000..a599108 --- /dev/null +++ b/internal/receiver/do.go @@ -0,0 +1,146 @@ +package receiver + +import ( + "io/fs" + "os" + "sync" + + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncstats" + "github.com/runpod/rsync/internal/rsyncwire" + "golang.org/x/sync/errgroup" +) + +func isTopDir(f *File) bool { + // TODO: once we check the f.Flags: + // if !f.FileMode().IsDir() { + // // non-directories can get the top_dir flag set, + // // but it must be ignored (only for protocol reasons). + // return false + // } + // return (f.Flags & TOP_DIR) != 0 + return f.Name == "." +} + +func (rt *Transfer) deleteFiles(fileList []*File) error { + if rt.IOErrors > 0 { + rt.Logger.Printf("IO error encountered, skipping file deletion") + return nil + } + + for _, f := range fileList { + if !isTopDir(f) { + continue + } + rt.Logger.Printf("deleting in %s", f.Name) + // Other rsync implementations generate a local file list and compare it + // with the remote file list, we re-implement the path→name mapping part + // of file list generation here. We could change it for consistency. + err := fs.WalkDir(rt.DestRoot.FS(), ".", func(path string, info fs.DirEntry, err error) error { + if err != nil { + return err + } + rt.Logger.Printf("WalkDir(%q)", path) + if findInFileList(fileList, path) { + return nil + } + if rt.Opts.Verbose { + rt.Logger.Printf(" deleting %s", path) + } + if rt.Opts.DryRun { + return nil + } + if err := rt.DestRoot.RemoveAll(path); err != nil { + rt.Logger.Printf(" deleting %s failed: %v", path, err) + // keep going + } + return fs.SkipDir // skip the just-deleted directory + }) + if err != nil { + if os.IsNotExist(err) { + return nil // destination does not exist, nothing to do + } + return err + } + } + return nil +} + +// rsync/main.c:do_recv +func (rt *Transfer) Do(c *rsyncwire.Conn, fileList []*File, noReport bool) (*rsyncstats.TransferStats, error) { + if rt.Opts.DeleteMode { + if err := rt.deleteFiles(fileList); err != nil { + return nil, err + } + } + + var eg errgroup.Group + // Wrap both, the generator and the receiver goroutine, in waitFor() calls + // to ensure we don’t block on the generator when the receiver returns an + // error, or vice versa (instead, return and let the goroutine finish in the + // background). + // waitFor calls f and waits for it to complete, but only until the specified + // context is cancelled. + var closeOnce sync.Once + closeOnErr := func(err error) error { + if err != nil { + closeOnce.Do(func() { c.Close() }) + } + return err + } + eg.Go(func() error { return closeOnErr(rt.GenerateFiles(fileList)) }) + eg.Go(func() error { return closeOnErr(rt.RecvFiles(fileList)) }) + if err := eg.Wait(); err != nil { + return nil, err + } + if rt.retouchDirPerms /* || rt.retouchDirTimes */ { + if err := rt.touchUpDirs(fileList); err != nil { + return nil, err + } + } + + var stats *rsyncstats.TransferStats + if !noReport { + var err error + stats, err = rt.report(c) + if err != nil { + return nil, err + } + } + + // send final goodbye message + if err := c.WriteInt32(-1); err != nil { + return nil, err + } + + return stats, nil +} + +// rsync/main.c:report +func (rt *Transfer) report(c *rsyncwire.Conn) (*rsyncstats.TransferStats, error) { + // read statistics: + // total bytes read (from network connection) + read, err := c.ReadInt64() + if err != nil { + return nil, err + } + // total bytes written (to network connection) + written, err := c.ReadInt64() + if err != nil { + return nil, err + } + // total size of files + size, err := c.ReadInt64() + if err != nil { + return nil, err + } + if rt.Opts.InfoGTE(rsyncopts.INFO_STATS, 1) { + rt.Logger.Printf("server sent stats: read=%d, written=%d, size=%d", read, written, size) + } + + return &rsyncstats.TransferStats{ + Read: read, + Written: written, + Size: size, + }, nil +} diff --git a/internal/receiver/flist.go b/internal/receiver/flist.go new file mode 100644 index 0000000..7cf2ea6 --- /dev/null +++ b/internal/receiver/flist.go @@ -0,0 +1,269 @@ +package receiver + +import ( + "fmt" + "io" + "io/fs" + "path/filepath" + "sort" + "time" + + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncchecksum" + "github.com/runpod/rsync/internal/rsyncopts" +) + +// rsync/flist.c:flist_sort_and_clean +func sortFileList(fileList []*File) { + sort.Slice(fileList, func(i, j int) bool { + return fileList[i].Name < fileList[j].Name + }) +} + +// rsync/receiver.c:delete_files +func findInFileList(fileList []*File, name string) bool { + i := sort.Search(len(fileList), func(i int) bool { + return fileList[i].Name >= name + }) + return i < len(fileList) && fileList[i].Name == name +} + +type File struct { + Name string + Length int64 + ModTime time.Time + Mode int32 + Uid int32 + Gid int32 + LinkTarget string + Rdev int32 + Checksum [rsyncchecksum.Size]byte +} + +// FileMode converts from the Linux permission bits to Go’s permission bits. +func (f *File) FileMode() fs.FileMode { + ret := fs.FileMode(f.Mode) & fs.ModePerm + + mode := f.Mode & rsync.S_IFMT + switch mode { + case rsync.S_IFCHR: + ret |= fs.ModeCharDevice + case rsync.S_IFBLK: + ret |= fs.ModeDevice + case rsync.S_IFIFO: + ret |= fs.ModeNamedPipe + case rsync.S_IFSOCK: + ret |= fs.ModeSocket + case rsync.S_IFLNK: + ret |= fs.ModeSymlink + case rsync.S_IFDIR: + ret |= fs.ModeDir + } + + return ret +} + +// rsync/flist.c:receive_file_entry +func (rt *Transfer) receiveFileEntry(flags uint16, last *File) (*File, error) { + f := &File{} + + var l1 int + if flags&rsync.XMIT_SAME_NAME != 0 { + l, err := rt.Conn.ReadByte() + if err != nil { + return nil, err + } + l1 = int(l) + } + + var l2 int + if flags&rsync.XMIT_LONG_NAME != 0 { + l, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + l2 = int(l) + } else { + l, err := rt.Conn.ReadByte() + if err != nil { + return nil, err + } + l2 = int(l) + } + // linux/limits.h + const PATH_MAX = 4096 + if l2 >= PATH_MAX-l1 { + const lastname = "" + return nil, fmt.Errorf("overflow: flags=0x%x l1=%d l2=%d lastname=%s", + flags, l1, l2, lastname) + } + b := make([]byte, l1+l2) + readb := b + if l1 > 0 { + copy(b, []byte(last.Name)) + readb = b[l1:] + } + if _, err := io.ReadFull(rt.Conn.Reader, readb); err != nil { + return nil, err + } + // TODO: does rsync’s clean_fname() and sanitize_path() combination do + // anything more than Go’s filepath.Clean()? + f.Name = filepath.Clean(string(b)) + + length, err := rt.Conn.ReadInt64() + if err != nil { + return nil, err + } + f.Length = length + + if flags&rsync.XMIT_SAME_TIME != 0 { + f.ModTime = last.ModTime + } else { + modTime, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + f.ModTime = time.Unix(int64(modTime), 0) + } + + if flags&rsync.XMIT_SAME_MODE != 0 { + f.Mode = last.Mode + } else { + mode, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + f.Mode = mode + } + + if rt.Opts.PreserveUid { + if flags&rsync.XMIT_SAME_UID != 0 { + f.Uid = last.Uid + } else { + uid, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + f.Uid = uid + } + } + + if rt.Opts.PreserveGid { + if flags&rsync.XMIT_SAME_GID != 0 { + f.Gid = last.Gid + } else { + gid, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + f.Gid = gid + } + } + + mode := f.Mode & rsync.S_IFMT + isDev := mode == rsync.S_IFCHR || mode == rsync.S_IFBLK + isSpecial := mode == rsync.S_IFIFO || mode == rsync.S_IFSOCK + isLink := mode == rsync.S_IFLNK + + if rt.Opts.PreserveDevices && (isDev || isSpecial) { + // TODO(protocol >= 28): rdev/major/minor handling + if flags&rsync.XMIT_SAME_RDEV_pre28 != 0 { + f.Rdev = last.Rdev + } else { + rdev, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + f.Rdev = rdev + } + } + + if rt.Opts.PreserveLinks && isLink { + length, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + b := make([]byte, length) + if _, err := io.ReadFull(rt.Conn.Reader, b); err != nil { + return nil, err + } + f.LinkTarget = string(b) + } + + if rt.Opts.AlwaysChecksum { + if _, err := io.ReadFull(rt.Conn.Reader, f.Checksum[:]); err != nil { + return nil, err + } + } + + return f, nil +} + +// rsync/flist.c:recv_file_list +func (rt *Transfer) ReceiveFileList() ([]*File, error) { + if rt.Opts.Progress { + fmt.Fprintln(rt.Env.Stdout, "receiving file list...") + fmt.Fprint(rt.Env.Stdout, "0 files to consider") + } + lastFileEntry := new(File) + var fileList []*File + for { + b, err := rt.Conn.ReadByte() + if err != nil { + return nil, err + } + if b == 0 { + break + } + flags := uint16(b) + // rt.Logger.Printf("flags: %x", flags) + // TODO(protocol >= 28): extended flags + + f, err := rt.receiveFileEntry(flags, lastFileEntry) + if err != nil { + return nil, err + } + lastFileEntry = f + // TODO: include depth in output? + if rt.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + rt.Logger.Printf("[Receiver] i=%d ? %s mode=%o len=%d uid=%d gid=%d flags=?", + len(fileList), + f.Name, + f.Mode, + f.Length, + f.Uid, + f.Gid) + } + fileList = append(fileList, f) + if rt.Opts.Progress && len(fileList)%100 == 0 { + fmt.Fprintf(rt.Env.Stdout, "\r%d files to consider", len(fileList)) + } + } + if rt.Opts.Progress { + fmt.Fprintf(rt.Env.Stdout, "\r%d files to consider\n", len(fileList)) + } + + sortFileList(fileList) + + if rt.Opts.PreserveUid || rt.Opts.PreserveGid { + // receive the uid/gid list + users, groups, err := rt.RecvIdList() + if err != nil { + return nil, err + } + rt.Users = users + rt.Groups = groups + } + + // read the i/o error flag + ioErrors, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 2) { + rt.Logger.Printf("ioErrors: %v", ioErrors) + } + rt.IOErrors = ioErrors + + return fileList, nil +} diff --git a/internal/receiver/generator.go b/internal/receiver/generator.go new file mode 100644 index 0000000..5c60520 --- /dev/null +++ b/internal/receiver/generator.go @@ -0,0 +1,361 @@ +package receiver + +import ( + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncchecksum" + "github.com/runpod/rsync/internal/rsynccommon" + "github.com/runpod/rsync/internal/rsyncopts" +) + +// rsync/generator.c:generate_files() +func (rt *Transfer) GenerateFiles(fileList []*File) error { + phase := 0 + for idx, f := range fileList { + if err := rt.recvGenerator(idx, f); err != nil { + return err + } + } + phase++ + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("generateFiles phase=%d", phase) + } + if err := rt.Conn.WriteInt32(-1); err != nil { + return err + } + + // TODO: re-do any files that failed + phase++ + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("generateFiles phase=%d", phase) + } + if err := rt.Conn.WriteInt32(-1); err != nil { + return err + } + + // NOTE: touchUpDirs is called from [Transfer.Do] + // so that both goroutines (generator and receiver) + // have finished before we set final permissions. + + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("generateFiles finished") + } + return nil +} + +func (rt *Transfer) touchUpDirs(fileList []*File) error { + for idx, f := range fileList { + if rt.Opts.DebugGTE(rsyncopts.DEBUG_TIME, 2) { + rt.Logger.Printf("touchUpDirs: %s (%d)", f.Name, idx) + } + mode := fs.FileMode(f.Mode) + if mode&rsync.S_IFMT != rsync.S_IFDIR { + continue // not a directory + } + if rt.Opts.DryRun { + continue + } + if mode&syscall.S_IWUSR > 0 { + continue // directory is writeable, no touchup needed + } + if err := rt.setPerms(f, mode); err != nil { + return err + } + } + return nil +} + +// rsync/generator.c:skip_file +func (rt *Transfer) skipFile(f *File, st os.FileInfo) (bool, error) { + if st.Size() != f.Length { + return false, nil + } + + if rt.Opts.AlwaysChecksum { + checksum, err := rsyncchecksum.RootChecksum(rt.DestRoot, f.Name) + if err != nil { + return false, err + } + return bytes.Equal(f.Checksum[:], checksum[:]), nil + } + + // TODO: size only + + if rt.Opts.IgnoreTimes { + return false, nil + } + + return modTimeEqual(st.ModTime(), f.ModTime), nil +} + +func modTimeEqual(a, b time.Time) bool { + a = a.Truncate(time.Second) + b = b.Truncate(time.Second) + return a.Equal(b) +} + +// rsync/rsync.c:set_perms +func (rt *Transfer) setPerms(f *File, mode fs.FileMode) error { + if rt.Opts.DryRun { + return nil + } + + st, err := rt.DestRoot.Lstat(f.Name) + if err != nil { + return err + } + + perm := mode & os.ModePerm + mode = mode & rsync.S_IFMT + if rt.Opts.PreserveTimes && + mode != rsync.S_IFLNK && + !modTimeEqual(st.ModTime(), f.ModTime) { + if err := rt.DestRoot.Chtimes(f.Name, f.ModTime, f.ModTime); err != nil { + return err + } + } + + _, err = rt.setUid(f, st) + if err != nil { + return err + } + + if mode != rsync.S_IFLNK { + if st.Mode().Perm() != perm { // only call Chmod if the permissions actually differ + if err := rt.DestRoot.Chmod(f.Name, perm); err != nil { + return err + } + } + } + + return nil +} + +// rsync/generator.c:recv_generator +func (rt *Transfer) recvGenerator(idx int, f *File) error { + if rt.listOnly() { + fmt.Fprintf(rt.Env.Stdout, "%s %11.0f %s %s\n", + f.FileMode().String(), + float64(f.Length), // TODO: rsync prints decimal separators + f.ModTime.Format("2006/01/02 15:04:05"), + f.Name) + return nil + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("recv_generator(f=%+v)", f) + } + + local := filepath.Join(rt.Dest, f.Name) + st, err := rt.DestRoot.Lstat(f.Name) + + mode := f.Mode & rsync.S_IFMT + if mode == rsync.S_IFDIR { + if rt.Opts.DryRun { + return nil + } + if err == nil && !st.IsDir() { + // A file (not a directory) with this name exists. Delete it so that + // we can create a directory instead. + if err := rt.DestRoot.Remove(f.Name); err != nil { + return fmt.Errorf("unlinking to make room for directory: %v", err) + } + err = fmt.Errorf("file removed") + } + if err != nil { + perm := fs.FileMode(f.Mode) & os.ModePerm + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("MkdirAll(%s, %v)", f.Name, perm) + } + if err := rt.DestRoot.MkdirAll(f.Name, perm); err != nil { + // TODO: EEXIST is okay + return err + } + // fallthrough to setPerms and return nil + } + mode := fs.FileMode(f.Mode) + if mode&syscall.S_IWUSR == 0 { + // The directory is lacking write permission, + // so we need to create it writeable as long as + // we are creating files inside that directory. + // GenerateFiles will fix permissions afterwards. + rt.retouchDirPerms = true + mode |= syscall.S_IWUSR + } + if err := rt.setPerms(f, mode); err != nil { + return err + } + return nil + } + + if rt.Opts.PreserveLinks && mode == rsync.S_IFLNK { + // TODO: safe_symlinks option + if err == nil { + // local file exists, verify target matches + if target, err := rt.DestRoot.Readlink(f.Name); err == nil { + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("existing target: %q", target) + } + if target == f.LinkTarget { + if err := rt.setPerms(f, fs.FileMode(f.Mode)); err != nil { + return err + } + return nil // skip + } + // fallthrough to create or replace the symlink + } + // fallthrough to create or replace the symlink + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("symlink %s -> %s", f.Name, f.LinkTarget) + } + if err := symlink(rt.DestRoot, f.LinkTarget, f.Name); err != nil { + return err + } + if err := rt.setPerms(f, fs.FileMode(f.Mode)); err != nil { + return err + } + return nil + } + + if rt.Opts.PreserveDevices && (mode == rsync.S_IFCHR || + mode == rsync.S_IFBLK || + mode == rsync.S_IFSOCK || + mode == rsync.S_IFIFO) { + if err := rt.createDevice(f, st); err != nil { + return err + } + return nil + } + + if rt.Opts.PreserveHardlinks { + // TODO: hard link check + } + + if !f.FileMode().IsRegular() { + // None of the Preserve* options is enabled, so just skip over + // non-regular files. + return nil + } + + requestFullFile := func() error { + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("requesting: %s", f.Name) + } + if err := rt.Conn.WriteInt32(int32(idx)); err != nil { + return err + } + if rt.Opts.DryRun { + return nil + } + var sh rsync.SumHead + if err := sh.WriteTo(rt.Conn); err != nil { + return err + } + return nil + } + + if os.IsNotExist(err) { + // Final absent: resume from a retained partial if one exists. + if rt.Opts.KeepPartial { + if pin, psize, ok := rt.openPartialBasis(f); ok { + defer pin.Close() + rt.Logger.Printf("resuming %s from partial (%d bytes)", f.Name, psize) + if err := rt.Conn.WriteInt32(int32(idx)); err != nil { + return err + } + return rt.generateAndSendSums(pin, psize) + } + } + return requestFullFile() + } + if err != nil { + return err + } + + if !st.Mode().IsRegular() { + // A non-regular file with this name exists. Delete it so that we can + // create our file instead. + if err := rt.DestRoot.Remove(f.Name); err != nil { + return fmt.Errorf("unlinking to make room for regular file: %v", err) + } + return requestFullFile() + } + + // TODO: update-only check + + skip, err := rt.skipFile(f, st) + if err != nil { + return err + } + if skip { + if rt.Opts.InfoGTE(rsyncopts.INFO_SKIP, 1) { + rt.Logger.Printf("skipping %s", local) + } + if err := rt.setPerms(f, fs.FileMode(f.Mode)); err != nil { + return err + } + return nil + } + + if rt.Opts.DryRun { + if err := rt.Conn.WriteInt32(int32(idx)); err != nil { + return err + } + + return nil + } + + // TODO: if deltas are disabled, request the file in full + + in, err := rt.DestRoot.Open(f.Name) + if err != nil { + rt.Logger.Printf("failed to open %s, continuing: %v", local, err) + return requestFullFile() + } + defer in.Close() + + if rt.Opts.DebugGTE(rsyncopts.DEBUG_GENR, 1) { + rt.Logger.Printf("sending sums for: %s", f.Name) + } + if err := rt.Conn.WriteInt32(int32(idx)); err != nil { + return err + } + + return rt.generateAndSendSums(in, st.Size()) +} + +// rsync/generator.c:generate_and_send_sums +func (rt *Transfer) generateAndSendSums(in *os.File, fileLen int64) error { + sh := rsynccommon.SumSizesSqroot(fileLen) + if err := sh.WriteTo(rt.Conn); err != nil { + return err + } + buf := make([]byte, int(sh.BlockLength)) + remaining := fileLen + for i := int32(0); i < sh.ChecksumCount; i++ { + n1 := min(int64(sh.BlockLength), remaining) + b := buf[:n1] + if _, err := io.ReadFull(in, b); err != nil { + return err + } + + sum1 := rsyncchecksum.Checksum1(b) + sum2 := rsyncchecksum.Checksum2(rt.Seed, b) + if err := rt.Conn.WriteInt32(int32(sum1)); err != nil { + return err + } + if _, err := rt.Conn.Writer.Write(sum2); err != nil { + return err + } + remaining -= n1 + } + return nil +} diff --git a/internal/receiver/generatormknod_darwin.go b/internal/receiver/generatormknod_darwin.go new file mode 100644 index 0000000..0918519 --- /dev/null +++ b/internal/receiver/generatormknod_darwin.go @@ -0,0 +1,59 @@ +package receiver + +import ( + "io/fs" + "os" + "path/filepath" + "syscall" + + "github.com/runpod/rsync" + "golang.org/x/sys/unix" +) + +func (rt *Transfer) createDevice(f *File, st fs.FileInfo) error { + local := filepath.Join(rt.Dest, f.Name) + perm := fs.FileMode(f.Mode) & os.ModePerm + mode := f.Mode & rsync.S_IFMT + switch mode { + case rsync.S_IFCHR: + if st != nil && st.Mode().Type()&os.ModeCharDevice != 0 { + return nil // file of correct type exists + } + return unix.Mknod(local, uint32(perm)|syscall.S_IFCHR, int(f.Rdev)) + + case rsync.S_IFBLK: + if st != nil && (st.Mode().Type()&os.ModeDevice != 0 || + st.Mode().Type()&os.ModeCharDevice != 0) { + return nil // file of correct type exists + } + + return unix.Mknod(local, uint32(perm)|syscall.S_IFBLK, int(f.Rdev)) + + case rsync.S_IFSOCK: + if st != nil && st.Mode().Type()&os.ModeSocket != 0 { + return nil // file of correct type exists + } + + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + if err != nil { + return err + } + + if err := unix.Bind(fd, &unix.SockaddrUnix{Name: local}); err != nil { + return err + } + + if err := unix.Close(fd); err != nil { + return err + } + return nil + + case rsync.S_IFIFO: + if st != nil && st.Mode().Type()&os.ModeNamedPipe != 0 { + return nil // file of correct type exists + } + + return unix.Mkfifo(local, uint32(perm)) + } + return nil +} diff --git a/internal/receiver/generatormknod_linux.go b/internal/receiver/generatormknod_linux.go new file mode 100644 index 0000000..bf2847e --- /dev/null +++ b/internal/receiver/generatormknod_linux.go @@ -0,0 +1,71 @@ +package receiver + +import ( + "fmt" + "io/fs" + "os" + "path/filepath" + "strconv" + "syscall" + + "github.com/runpod/rsync" + "golang.org/x/sys/unix" +) + +func (rt *Transfer) createDevice(f *File, st fs.FileInfo) error { + base := filepath.Base(f.Name) + parentDir, err := rt.DestRoot.OpenFile(filepath.Dir(f.Name), 0, 0) + if err != nil { + return fmt.Errorf("Open(parent(%s)): %v", f.Name, err) + } + defer parentDir.Close() + perm := fs.FileMode(f.Mode) & os.ModePerm + mode := f.Mode & rsync.S_IFMT + switch mode { + case rsync.S_IFCHR: + if st != nil && st.Mode().Type()&os.ModeCharDevice != 0 { + return nil // file of correct type exists + } + return unix.Mknodat(int(parentDir.Fd()), base, uint32(perm)|syscall.S_IFCHR, int(f.Rdev)) + + case rsync.S_IFBLK: + if st != nil && (st.Mode().Type()&os.ModeDevice != 0 || + st.Mode().Type()&os.ModeCharDevice != 0) { + return nil // file of correct type exists + } + + return unix.Mknodat(int(parentDir.Fd()), base, uint32(perm)|syscall.S_IFBLK, int(f.Rdev)) + + case rsync.S_IFSOCK: + if st != nil && st.Mode().Type()&os.ModeSocket != 0 { + return nil // file of correct type exists + } + + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + if err != nil { + return err + } + + // The parent dir is safely resolved through *os.Root, + // so we skip path resolution by constructing a path + // from a known-safe prefix (/proc/self/fd/) + // and a basename (not a path!). + local := filepath.Join("/proc/self/fd", strconv.Itoa(int(parentDir.Fd())), base) + if err := unix.Bind(fd, &unix.SockaddrUnix{Name: local}); err != nil { + return err + } + + if err := unix.Close(fd); err != nil { + return err + } + return nil + + case rsync.S_IFIFO: + if st != nil && st.Mode().Type()&os.ModeNamedPipe != 0 { + return nil // file of correct type exists + } + + return unix.Mkfifoat(int(parentDir.Fd()), base, uint32(perm)) + } + return nil +} diff --git a/internal/receiver/generatormknodstub.go b/internal/receiver/generatormknodstub.go new file mode 100644 index 0000000..1b5457a --- /dev/null +++ b/internal/receiver/generatormknodstub.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin + +package receiver + +import "io/fs" + +func (rt *Transfer) createDevice(*File, fs.FileInfo) error { + return nil +} diff --git a/internal/receiver/generatorsymlink.go b/internal/receiver/generatorsymlink.go new file mode 100644 index 0000000..81b0ed2 --- /dev/null +++ b/internal/receiver/generatorsymlink.go @@ -0,0 +1,13 @@ +//go:build linux || darwin + +package receiver + +import ( + "os" + + "github.com/google/renameio/v2" +) + +func symlink(root *os.Root, oldname, newname string) error { + return renameio.SymlinkRoot(root, oldname, newname) +} diff --git a/internal/receiver/generatorsymlink_windows.go b/internal/receiver/generatorsymlink_windows.go new file mode 100644 index 0000000..ba97dd4 --- /dev/null +++ b/internal/receiver/generatorsymlink_windows.go @@ -0,0 +1,12 @@ +//go:build windows + +package receiver + +import "os" + +func symlink(destroot *os.Root, oldname, newname string) error { + if err := destroot.Remove(newname); err != nil && !os.IsNotExist(err) { + return err + } + return destroot.Symlink(oldname, newname) +} diff --git a/internal/receiver/generatoruid.go b/internal/receiver/generatoruid.go new file mode 100644 index 0000000..55df596 --- /dev/null +++ b/internal/receiver/generatoruid.go @@ -0,0 +1,62 @@ +//go:build linux || darwin + +package receiver + +import ( + "io/fs" + "os" + "os/user" + "strconv" + "syscall" +) + +var amRoot = os.Getuid() == 0 + +var inGroup = func() map[uint32]bool { + m := make(map[uint32]bool) + u, err := user.Current() + if err != nil { + return m + } + gids, err := u.GroupIds() + if err != nil { + return m + } + for _, gidString := range gids { + gid64, err := strconv.ParseInt(gidString, 0, 64) + if err != nil { + return m + } + m[uint32(gid64)] = true + } + return m +}() + +func (rt *Transfer) setUid(f *File, st fs.FileInfo) (fs.FileInfo, error) { + stt := st.Sys().(*syscall.Stat_t) + + changeUid := rt.Opts.PreserveUid && + amRoot && + stt.Uid != uint32(f.Uid) + + changeGid := rt.Opts.PreserveGid && + (amRoot || inGroup[uint32(f.Gid)]) && + stt.Gid != uint32(f.Gid) + + if !changeUid && !changeGid { + return st, nil + } + + uid := stt.Uid + if changeUid { + uid = uint32(f.Uid) + } + gid := stt.Gid + if changeGid { + gid = uint32(f.Gid) + } + if err := rt.DestRoot.Lchown(f.Name, int(uid), int(gid)); err != nil { + return nil, err + } + return rt.DestRoot.Lstat(f.Name) +} diff --git a/internal/receiver/generatoruidstub.go b/internal/receiver/generatoruidstub.go new file mode 100644 index 0000000..61cd632 --- /dev/null +++ b/internal/receiver/generatoruidstub.go @@ -0,0 +1,9 @@ +//go:build !linux && !darwin + +package receiver + +import "io/fs" + +func (rt *Transfer) setUid(_ *File, st fs.FileInfo) (fs.FileInfo, error) { + return st, nil +} diff --git a/internal/receiver/receiver.go b/internal/receiver/receiver.go new file mode 100644 index 0000000..fdf7f9d --- /dev/null +++ b/internal/receiver/receiver.go @@ -0,0 +1,257 @@ +package receiver + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "github.com/mmcloughlin/md4" + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncopts" +) + +// rsync/receiver.c:recv_files +func (rt *Transfer) RecvFiles(fileList []*File) error { + phase := 0 + for { + idx, err := rt.Conn.ReadInt32() + if err != nil { + return err + } + if idx == -1 { + if phase == 0 { + phase++ + if rt.Opts.DebugGTE(rsyncopts.DEBUG_RECV, 1) { + rt.Logger.Printf("recvFiles phase=%d", phase) + } + // TODO: send done message + continue + } + break + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_RECV, 1) { + rt.Logger.Printf("receiving file idx=%d: %+v", idx, fileList[idx]) + } + if rt.Opts.Progress { + fmt.Fprintln(rt.Env.Stdout, fileList[idx].Name) + } + if err := rt.recvFile1(fileList[idx]); err != nil { + return err + } + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_RECV, 1) { + rt.Logger.Printf("recvFiles finished") + } + return nil +} + +func (rt *Transfer) recvFile1(f *File) error { + if rt.Opts.DryRun { + if !rt.Opts.Server { + fmt.Fprintln(rt.Env.Stdout, f.Name) + } + return nil + } + + localFile, err := rt.openLocalFile(f) + if err != nil && !os.IsNotExist(err) { + rt.Logger.Printf("opening local file failed, continuing: %v", err) + } + defer localFile.Close() + if err := rt.receiveData(f, localFile); err != nil { + return err + } + return nil +} + +func (rt *Transfer) openLocalFile(f *File) (*os.File, error) { + name := f.Name + in, err := rt.DestRoot.Open(name) + if err != nil && rt.Opts.KeepPartial && os.IsNotExist(err) { + // Final absent: fall back to the retained partial as the delta basis. + name = f.Name + partialSuffix + in, err = rt.DestRoot.Open(name) + } + if err != nil { + return nil, err + } + usedPartial := name != f.Name + + st, err := in.Stat() + if err != nil { + return nil, err + } + + if st.IsDir() { + return nil, fmt.Errorf("%s is a directory", filepath.Join(rt.Dest, name)) + } + + if !st.Mode().IsRegular() { + return nil, nil + } + + if !rt.Opts.PreservePerms && !usedPartial { + // If the file exists already and we are not preserving permissions, + // then act as though the remote sent us the existing permissions. + // Skipped for a partial basis, whose temp perms aren't the target's. + f.Mode = int32(st.Mode().Perm()) + } + + return in, nil +} + +// rsync/receiver.c:receive_data +func (rt *Transfer) receiveData(f *File, localFile *os.File) error { + rt.Progress.Reset(uint64(f.Length)) + var sh rsync.SumHead + if err := sh.ReadFrom(rt.Conn); err != nil { + return err + } + + if rt.Opts.DebugGTE(rsyncopts.DEBUG_DELTASUM, 1) { + rt.Logger.Printf("creating %s", filepath.Join(rt.Dest, f.Name)) + } + + // Default path: renameio (atomic temp+rename, discard on failure). With + // KeepPartial an interrupt instead retains ".partial" to resume from. + var w io.Writer + var commit func() error + var abort func() + // discardPartial drops the bytes on verification failure instead of keeping + // a corrupt partial as a future basis. + discardPartial := false + + if rt.Opts.KeepPartial { + tmpName := f.Name + partialSuffix + ".tmp" + partialName := f.Name + partialSuffix + tmp, err := rt.DestRoot.OpenFile(tmpName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + w = tmp + commit = func() error { + if rt.Opts.DoFsync { + if err := tmp.Sync(); err != nil { + return err + } + } + if err := tmp.Close(); err != nil { + return err + } + if err := rt.DestRoot.Rename(tmpName, f.Name); err != nil { + return err + } + _ = rt.DestRoot.Remove(partialName) + return nil + } + abort = func() { + _ = tmp.Close() + if discardPartial { + _ = rt.DestRoot.Remove(tmpName) + _ = rt.DestRoot.Remove(partialName) + return + } + rt.retainPartial(tmpName, partialName) + } + } else { + out, err := newPendingFile(rt.DestRoot, f.Name, rt.Opts.DoFsync) + if err != nil { + return err + } + w = out + commit = out.CloseAtomicallyReplace + abort = func() { _ = out.Cleanup() } + } + + committed := false + defer func() { + if !committed { + abort() + } + }() + + h := md4.New() + binary.Write(h, binary.LittleEndian, rt.Seed) + + wr := io.MultiWriter(w, h) + + offset := 0 + for { + token, data, err := rt.recvToken() + if err != nil { + return err + } + if token == 0 { + break + } + if rt.Opts.Progress && !rt.Opts.Server { + rt.Progress.MaybeShow(uint64(offset), false) + if offset == 0 { + defer func() { + rt.Progress.MaybeShow(uint64(offset), true) + }() + } + } + if token > 0 { + n, err := wr.Write(data) + if err != nil { + return err + } + offset += n + continue + } + if localFile == nil { + return fmt.Errorf("BUG: local file %s not open for copying chunk", f.Name) + } + token = -(token + 1) + offset2 := int64(token) * int64(sh.BlockLength) + dataLen := sh.BlockLength + if token == sh.ChecksumCount-1 && sh.RemainderLength != 0 { + dataLen = sh.RemainderLength + } + data = make([]byte, dataLen) + if _, err := localFile.ReadAt(data, offset2); err != nil { + return err + } + + n, err := wr.Write(data) + if err != nil { + return err + } + offset += n + } + localSum := h.Sum(nil) + remoteSum := make([]byte, len(localSum)) + if _, err := io.ReadFull(rt.Conn.Reader, remoteSum); err != nil { + return err + } + if !bytes.Equal(localSum, remoteSum) { + discardPartial = true + return fmt.Errorf("file corruption in %s", f.Name) + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_DELTASUM, 1) { + rt.Logger.Printf("checksum %x matches!", localSum) + } + + if localFile != nil { + // Close the file earlier than the calling function’s deferred Close(), + // so that we can rename files on Windows, which fails as long + // as there are any open file handles. + localFile.Close() + } + + if err := commit(); err != nil { + return err + } + committed = true + + if err := rt.setPerms(f, fs.FileMode(f.Mode)); err != nil { + return err + } + + return nil +} diff --git a/internal/receiver/receiverpartial.go b/internal/receiver/receiverpartial.go new file mode 100644 index 0000000..932831d --- /dev/null +++ b/internal/receiver/receiverpartial.go @@ -0,0 +1,46 @@ +package receiver + +import "os" + +// partialSuffix is the sidecar a --partial transfer retains on interrupt. +const partialSuffix = ".partial" + +// openPartialBasis opens ".partial" as the delta basis for f. ok is false +// when no usable (regular, non-empty) partial exists. +func (rt *Transfer) openPartialBasis(f *File) (in *os.File, size int64, ok bool) { + in, err := rt.DestRoot.Open(f.Name + partialSuffix) + if err != nil { + return nil, 0, false + } + st, err := in.Stat() + if err != nil || !st.Mode().IsRegular() || st.Size() == 0 { + in.Close() + return nil, 0, false + } + return in, st.Size(), true +} + +// retainPartial promotes the temp to ".partial", keeping whichever is +// longer so the basis only advances. Names are relative to the destination root. +func (rt *Transfer) retainPartial(tmpName, partialName string) { + tmpSize := rt.rootFileSize(tmpName) + if tmpSize < 0 { + return // nothing written this attempt + } + if tmpSize < rt.rootFileSize(partialName) { + _ = rt.DestRoot.Remove(tmpName) // existing partial is more complete + return + } + if err := rt.DestRoot.Rename(tmpName, partialName); err != nil { + rt.Logger.Printf("failed to retain partial %s: %v", partialName, err) + _ = rt.DestRoot.Remove(tmpName) + } +} + +func (rt *Transfer) rootFileSize(name string) int64 { + st, err := rt.DestRoot.Stat(name) + if err != nil { + return -1 + } + return st.Size() +} diff --git a/internal/receiver/receiverpartial_test.go b/internal/receiver/receiverpartial_test.go new file mode 100644 index 0000000..8c5e9f6 --- /dev/null +++ b/internal/receiver/receiverpartial_test.go @@ -0,0 +1,116 @@ +package receiver + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/runpod/rsync/internal/log" +) + +func newTestTransfer(t *testing.T, dest string) *Transfer { + t.Helper() + root, err := os.OpenRoot(dest) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { root.Close() }) + return &Transfer{ + Logger: log.New(io.Discard), + Dest: dest, + DestRoot: root, + } +} + +func writeN(t *testing.T, path string, n int) { + t.Helper() + if err := os.WriteFile(path, make([]byte, n), 0644); err != nil { + t.Fatal(err) + } +} + +func sizeOf(t *testing.T, path string) int64 { + t.Helper() + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + return st.Size() +} + +// TestRetainPartialKeepsLonger checks the basis only advances: a longer temp +// replaces the partial, a shorter one is dropped, and a missing temp is a no-op. +func TestRetainPartialKeepsLonger(t *testing.T) { + const tmpName, partialName = "f.tgz.partial.tmp", "f.tgz.partial" + + t.Run("longer temp replaces partial", func(t *testing.T) { + dir := t.TempDir() + rt := newTestTransfer(t, dir) + writeN(t, filepath.Join(dir, partialName), 100) + writeN(t, filepath.Join(dir, tmpName), 250) + + rt.retainPartial(tmpName, partialName) + + if _, err := os.Stat(filepath.Join(dir, tmpName)); !os.IsNotExist(err) { + t.Errorf("temp should be renamed away, stat err = %v", err) + } + if got := sizeOf(t, filepath.Join(dir, partialName)); got != 250 { + t.Errorf("partial size = %d, want 250 (longer temp)", got) + } + }) + + t.Run("shorter temp is discarded", func(t *testing.T) { + dir := t.TempDir() + rt := newTestTransfer(t, dir) + writeN(t, filepath.Join(dir, partialName), 500) + writeN(t, filepath.Join(dir, tmpName), 120) + + rt.retainPartial(tmpName, partialName) + + if _, err := os.Stat(filepath.Join(dir, tmpName)); !os.IsNotExist(err) { + t.Errorf("shorter temp should be removed, stat err = %v", err) + } + if got := sizeOf(t, filepath.Join(dir, partialName)); got != 500 { + t.Errorf("partial size = %d, want 500 (kept more complete partial)", got) + } + }) + + t.Run("missing temp leaves partial intact", func(t *testing.T) { + dir := t.TempDir() + rt := newTestTransfer(t, dir) + writeN(t, filepath.Join(dir, partialName), 300) + + rt.retainPartial(tmpName, partialName) + + if got := sizeOf(t, filepath.Join(dir, partialName)); got != 300 { + t.Errorf("partial size = %d, want 300 (unchanged)", got) + } + }) +} + +// TestOpenPartialBasis checks a regular non-empty partial is opened, while a +// missing or empty one reports ok=false. +func TestOpenPartialBasis(t *testing.T) { + dir := t.TempDir() + rt := newTestTransfer(t, dir) + + if _, _, ok := rt.openPartialBasis(&File{Name: "missing.tgz"}); ok { + t.Error("absent partial should report ok=false") + } + + writeN(t, filepath.Join(dir, "empty.tgz"+partialSuffix), 0) + if _, _, ok := rt.openPartialBasis(&File{Name: "empty.tgz"}); ok { + t.Error("empty partial should report ok=false") + } + + writeN(t, filepath.Join(dir, "good.tgz"+partialSuffix), 4096) + in, sz, ok := rt.openPartialBasis(&File{Name: "good.tgz"}) + if !ok { + t.Fatal("regular non-empty partial should report ok=true") + } + defer in.Close() + if sz != 4096 { + t.Errorf("basis size = %d, want 4096", sz) + } +} diff --git a/internal/receiver/receiverrenameio.go b/internal/receiver/receiverrenameio.go new file mode 100644 index 0000000..77a8009 --- /dev/null +++ b/internal/receiver/receiverrenameio.go @@ -0,0 +1,89 @@ +package receiver + +import ( + "math/rand/v2" + "os" + "path/filepath" + "strconv" +) + +// openTempFileRoot creates a randomly named file in root and returns an open +// handle. It is similar to os.CreateTemp except that the directory must be +// given, the file permissions can be controlled and patterns in the name are +// not supported. The name is always suffixed with a random number. +func openTempFileRoot(root *os.Root, name string, perm os.FileMode) (string, *os.File, error) { + prefix := name + + for attempt := 0; ; { + // Generate a reasonably random name which is unlikely to already + // exist. O_EXCL ensures that existing files generate an error. + name := prefix + strconv.FormatInt(rand.Int64(), 10) + + f, err := root.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, perm) + if !os.IsExist(err) { + return name, f, err + } + + if attempt++; attempt > 10000 { + return "", nil, &os.PathError{ + Op: "tempfile", + Path: name, + Err: os.ErrExist, + } + } + } +} + +type pendingFile struct { + root *os.Root + tmpname string + fn string + f *os.File + sync bool +} + +func newPendingFile(root *os.Root, fn string, sync bool) (*pendingFile, error) { + tmpname, f, err := openTempFileRoot(root, "."+filepath.Base(fn), 0o600) + if err != nil { + return nil, err + } + return &pendingFile{ + root: root, + tmpname: tmpname, + fn: fn, + f: f, + sync: sync, + }, nil +} + +func (p *pendingFile) Name() string { + return p.fn +} + +func (p *pendingFile) Write(buf []byte) (n int, _ error) { + return p.f.Write(buf) +} + +func (p *pendingFile) CloseAtomicallyReplace() error { + if p.sync { + // fsync was requested + if err := p.f.Sync(); err != nil { + return err + } + } + if err := p.f.Close(); err != nil { + return err + } + if err := p.root.Rename(p.tmpname, p.fn); err != nil { + return err + } + return nil +} + +func (p *pendingFile) Cleanup() error { + err := p.f.Close() + if err := p.root.Remove(p.tmpname); err != nil { + return err + } + return err +} diff --git a/internal/receiver/token.go b/internal/receiver/token.go new file mode 100644 index 0000000..e9fd5af --- /dev/null +++ b/internal/receiver/token.go @@ -0,0 +1,20 @@ +package receiver + +import "io" + +// rsync/token.c:recvToken +func (rt *Transfer) recvToken() (token int32, data []byte, _ error) { + var err error + token, err = rt.Conn.ReadInt32() + if err != nil { + return 0, nil, err + } + if token <= 0 { + return token, nil, nil + } + data = make([]byte, int(token)) + if _, err := io.ReadFull(rt.Conn.Reader, data); err != nil { + return 0, nil, err + } + return token, data, nil +} diff --git a/internal/receiver/transfer.go b/internal/receiver/transfer.go new file mode 100644 index 0000000..af105d1 --- /dev/null +++ b/internal/receiver/transfer.go @@ -0,0 +1,57 @@ +package receiver + +import ( + "os" + + "github.com/runpod/rsync/internal/log" + "github.com/runpod/rsync/internal/progress" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncwire" +) + +// TransferOpts is a subset of Opts which is required for implementing a receiver. +type TransferOpts struct { + Verbose bool + DryRun bool + Server bool + Progress bool + + DeleteMode bool + PreserveGid bool + PreserveUid bool + PreserveLinks bool + PreservePerms bool + PreserveDevices bool + PreserveSpecials bool + PreserveTimes bool + PreserveHardlinks bool + IgnoreTimes bool + AlwaysChecksum bool + DoFsync bool + + InfoGTE func(rsyncopts.InfoLevel, uint16) bool + DebugGTE func(rsyncopts.DebugLevel, uint16) bool + + KeepPartial bool +} + +type Transfer struct { + // config + Logger log.Logger + Opts *TransferOpts + Dest string + DestRoot *os.Root + Env *rsyncos.Env + Progress progress.Printer + + // state + Conn *rsyncwire.Conn + Seed int32 + IOErrors int32 + Users map[int32]mapping + Groups map[int32]mapping + retouchDirPerms bool +} + +func (rt *Transfer) listOnly() bool { return rt.Dest == "" } diff --git a/internal/receiver/uidlist.go b/internal/receiver/uidlist.go new file mode 100644 index 0000000..cdd85ae --- /dev/null +++ b/internal/receiver/uidlist.go @@ -0,0 +1,91 @@ +package receiver + +import ( + "io" + "os/user" + "strconv" + + "github.com/runpod/rsync/internal/rsyncopts" +) + +type mapping struct { + Name string + LocalId int32 +} + +func (rt *Transfer) recvIdMapping1(localId func(id int32, name string) int32) (map[int32]mapping, error) { + idMapping := make(map[int32]mapping) + for { + id, err := rt.Conn.ReadInt32() + if err != nil { + return nil, err + } + if id == 0 { + break + } + length, err := rt.Conn.ReadByte() + if err != nil { + return nil, err + } + name := make([]byte, length) + if _, err := io.ReadFull(rt.Conn.Reader, name); err != nil { + return nil, err + } + idMapping[id] = mapping{ + Name: string(name), + LocalId: localId(id, string(name)), + } + } + return idMapping, nil +} + +// rsync/uidlist.c:recv_id_list +func (rt *Transfer) RecvIdList() (users map[int32]mapping, groups map[int32]mapping, _ error) { + if rt.Opts.PreserveUid { + var err error + users, err = rt.recvIdMapping1(func(remoteUid int32, remoteUsername string) int32 { + u, err := user.Lookup(remoteUsername) + if err != nil { + return remoteUid + } + uid, err := strconv.ParseInt(u.Uid, 0, 32) + if err != nil { + return remoteUid + } + return int32(uid) + }) + if err != nil { + return nil, nil, err + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 2) { + for remoteUid, mapping := range users { + rt.Logger.Printf("remote uid %d(%s) maps to local uid %d", remoteUid, mapping.Name, mapping.LocalId) + } + } + } + + if rt.Opts.PreserveGid { + var err error + groups, err = rt.recvIdMapping1(func(remoteGid int32, remoteGroupname string) int32 { + g, err := user.LookupGroup(remoteGroupname) + if err != nil { + return remoteGid + } + gid, err := strconv.ParseInt(g.Gid, 0, 32) + if err != nil { + return remoteGid + } + return int32(gid) + }) + if err != nil { + return nil, nil, err + } + if rt.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 2) { + for remoteGid, mapping := range groups { + rt.Logger.Printf("remote gid %d(%s) maps to local gid %d", remoteGid, mapping.Name, mapping.LocalId) + } + } + } + + return users, groups, nil +} diff --git a/internal/restrict/restrict_linux.go b/internal/restrict/restrict_linux.go new file mode 100644 index 0000000..950aee1 --- /dev/null +++ b/internal/restrict/restrict_linux.go @@ -0,0 +1,61 @@ +// Package restrict can be used to restrict further file system access of the +// process if the operating system provides an API for that. +package restrict + +import ( + "fmt" + "log" + "os" + + "github.com/landlock-lsm/go-landlock/landlock" +) + +// ExtraHook is set when testing to make the landlock rule set more permissive. +var ExtraHook func() []landlock.Rule + +func MaybeFileSystem(roDirsOrFiles []string, rwDirs []string) error { + re := ExtraHook + if re == nil { + re = func() []landlock.Rule { + return nil + } + } + var roDirs, roFiles []string + for _, fn := range roDirsOrFiles { + st, err := os.Stat(fn) + if err != nil { + return err + } + if st.IsDir() { + roDirs = append(roDirs, fn) + } else { + roFiles = append(roFiles, fn) + } + } + log.Printf("setting up landlock ACL (paths ro: %q, paths rw: %q)", roDirs, rwDirs) + err := landlock.V3.BestEffort().RestrictPaths( + append(re(), []landlock.Rule{ + landlock.RODirs(defaultRoDirs...).IgnoreIfMissing(), + landlock.RODirs(roDirs...).IgnoreIfMissing(), + landlock.ROFiles(roFiles...).IgnoreIfMissing(), + landlock.RWDirs(rwDirs...).WithRefer(), + }...)...) + if err != nil { + return fmt.Errorf("landlock: %v", err) + } + + // Check whether landlock worked and print the result to the log. + // + // We use /sys because that path should never be required + // for regular functioning, yet is standard enough to be present + // on all supported Linux versions (including gokrazy). + const verifyPath = "/sys" + _, err = os.ReadDir(verifyPath) + if err == nil { + log.Printf("landlock seems ineffective: readdir(%s) unexpectedly worked!", verifyPath) + } else { + log.Printf("landlock verified: readdir(%s) = %v", verifyPath, err) + } + + return nil +} diff --git a/internal/restrict/restrict_other.go b/internal/restrict/restrict_other.go new file mode 100644 index 0000000..d60a51c --- /dev/null +++ b/internal/restrict/restrict_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package restrict + +import "github.com/landlock-lsm/go-landlock/landlock" + +// TODO: implement support for OpenBSD unveil(2)? + +var ExtraHook func() []landlock.Rule + +func MaybeFileSystem(_, _ []string) error { return nil } diff --git a/internal/restrict/restrictdefault_gokrazy.go b/internal/restrict/restrictdefault_gokrazy.go new file mode 100644 index 0000000..a1dea5e --- /dev/null +++ b/internal/restrict/restrictdefault_gokrazy.go @@ -0,0 +1,12 @@ +//go:build gokrazy + +package restrict + +var defaultRoDirs = []string{ + // See restrictdefault_others.go for rationale + "/etc", + // On systems with a read-only root file systems (like gokrazy), + // /etc/resolv.conf is a symlink to /tmp/resolv.conf, + // so we also need read-only access to /tmp. + "/tmp", +} diff --git a/internal/restrict/restrictdefault_others.go b/internal/restrict/restrictdefault_others.go new file mode 100644 index 0000000..226202d --- /dev/null +++ b/internal/restrict/restrictdefault_others.go @@ -0,0 +1,22 @@ +//go:build !gokrazy + +package restrict + +var defaultRoDirs = []string{ + // rsync needs /etc/passwd and /etc/group for user/group lookup. + // + // As of Go 1.24, the net package Go resolver reads + // the following DNS configurations files: + // + // - /etc/resolv.conf + // - /etc/hosts + // - /etc/services + // - /etc/nsswitch.conf + // + // Because the /etc/resolv.conf file might be re-created (by DHCP + // clients, Tailscale, or similar), we need to provide the entire + // /etc directory instead of individual files. Otherwise, the + // program seems to work at first and then fails DNS resolution + // after a while. + "/etc", +} diff --git a/internal/rsyncchecksum/checksum_test.go b/internal/rsyncchecksum/checksum_test.go new file mode 100644 index 0000000..1e75351 --- /dev/null +++ b/internal/rsyncchecksum/checksum_test.go @@ -0,0 +1,73 @@ +package rsyncchecksum_test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/runpod/rsync/internal/rsyncchecksum" +) + +func constructLargeDataFile(headPattern, bodyPattern, endPattern []byte) []byte { + // create large data file in source directory to be copied + head := bytes.Repeat(headPattern, 1*1024*1024) + body := bytes.Repeat(bodyPattern, 1*1024*1024) + end := bytes.Repeat(endPattern, 1*1024*1024) + return append(append(head, body...), end...) +} + +func writeLargeDataFile(t *testing.T, source string, headPattern, bodyPattern, endPattern []byte) { + // create large data file in source directory to be copied + content := constructLargeDataFile(headPattern, bodyPattern, endPattern) + large := filepath.Join(source, "large-data-file") + if err := os.MkdirAll(filepath.Dir(large), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(large, content, 0644); err != nil { + t.Fatal(err) + } +} + +func TestSyncExtended(t *testing.T) { + tmp := t.TempDir() + source := filepath.Join(tmp, "source") + + writeLargeDataFile(t, source, []byte{0x11}, []byte{0xbb}, []byte{0xee}) + + // These values are taken from the rsync debug output: + const k = 1768 + want := make([]uint32, 1780) + for i := 0; i <= 592; i++ { + want[i] = 0xa5d47568 + } + want[593] = 0x23645688 + for i := 594; i <= 1185; i++ { + want[i] = 0x8c1c2378 + } + want[1186] = 0x12504720 + for i := 1187; i <= 1778; i++ { + want[i] = 0x7d9883b0 + } + want[1779] = 0x61b8dff0 + + sourceLarge := filepath.Join(source, "large-data-file") + f, err := os.Open(sourceLarge) + if err != nil { + t.Fatal(err) + } + defer f.Close() + buf := make([]byte, k) + for idx, wantChecksum := range want { + n, err := f.Read(buf) + if err != nil { + t.Fatal(err) + } + + chunk := buf[:n] + sum := rsyncchecksum.Checksum1(chunk) + if sum != wantChecksum { + t.Fatalf("checksum calculation error: got %08x, want %08x (idx %d), chunk: %#v", sum, wantChecksum, idx, chunk) + } + } +} diff --git a/internal/rsyncchecksum/rsyncchecksum.go b/internal/rsyncchecksum/rsyncchecksum.go new file mode 100644 index 0000000..d3c9b1d --- /dev/null +++ b/internal/rsyncchecksum/rsyncchecksum.go @@ -0,0 +1,77 @@ +package rsyncchecksum + +import ( + "encoding/binary" + "io" + "os" + + "github.com/mmcloughlin/md4" +) + +func Tag2(s1, s2 uint16) uint16 { + return (((s1) + (s2)) & 0xFFFF) +} + +func Tag(sum uint32) uint16 { + return Tag2(uint16(sum&0xFFFF), uint16(sum>>16)) +} + +// SignExtend mirrors how C converts from (signed char) to uint32, i.e. using +// sign extension. get_checksum1 treats the buffer as (signed char*) instead of +// (unsigned char*), which likely was not a conscious choice, but here we are. +// +// This function is exported for use in the rolling checksum in match.go. +func SignExtend(b byte) uint32 { + val := uint32(b) + return uint32(int32(val<<24) >> 24) +} + +func Checksum1(buf []byte) uint32 { + bufLen := len(buf) + var s1, s2 uint32 + var i int + + if bufLen > 4 { + for i = 0; i < (bufLen - 4); i += 4 { + s2 += 4*(s1+SignExtend(buf[i])) + + 3*SignExtend(buf[i+1]) + + 2*SignExtend(buf[i+2]) + + SignExtend(buf[i+3]) + s1 += SignExtend(buf[i+0]) + + SignExtend(buf[i+1]) + + SignExtend(buf[i+2]) + + SignExtend(buf[i+3]) + } + } + for ; i < bufLen; i++ { + s1 += SignExtend(buf[i]) + s2 += s1 + } + return (s1 & 0xffff) + (s2 << 16) +} + +func Checksum2(seed int32, buf []byte) []byte { + h := md4.New() + h.Write(buf) + binary.Write(h, binary.LittleEndian, seed) + return h.Sum(nil) +} + +func ReaderChecksum(r io.Reader) ([]byte, error) { + h := md4.New() + if _, err := io.Copy(h, r); err != nil { + return nil, err + } + return h.Sum(nil), nil +} + +func RootChecksum(root *os.Root, fn string) ([]byte, error) { + f, err := root.Open(fn) + if err != nil { + return nil, err + } + defer f.Close() + return ReaderChecksum(f) +} + +const Size = md4.Size diff --git a/internal/rsynccommon/rsynccommon.go b/internal/rsynccommon/rsynccommon.go new file mode 100644 index 0000000..142e417 --- /dev/null +++ b/internal/rsynccommon/rsynccommon.go @@ -0,0 +1,37 @@ +// Package rsynccommon contains functionality that both the sender and the +// receiver implementation need. +package rsynccommon + +import ( + "math" + + "github.com/runpod/rsync" +) + +const blockSize = 700 // rsync/rsync.h + +// Corresponds to rsync/generator.c:sum_sizes_sqroot +func SumSizesSqroot(contentLen int64) rsync.SumHead { + // * The block size is a rounded square root of file length. + + // The block size algorithm plays a crucial role in the protocol efficiency. In general, the block size is the rounded square root of the total file size. The minimum block size, however, is 700 B. Otherwise, the square root computation is simply sqrt(3) followed by ceil(3) + + // For reasons unknown, the square root result is rounded up to the nearest multiple of eight. + + // TODO: round this + blockLength := max(int32(math.Sqrt(float64(contentLen))), blockSize) + + // * The checksum size is determined according to: + // * blocksum_bits = BLOCKSUM_EXP + 2*log2(file_len) - log2(block_len) + // * provided by Donovan Baarda which gives a probability of rsync + // * algorithm corrupting data and falling back using the whole md4 + // * checksums. + const checksumLength = 16 // TODO? + + return rsync.SumHead{ + ChecksumCount: int32((contentLen + (int64(blockLength) - 1)) / int64(blockLength)), + RemainderLength: int32(contentLen % int64(blockLength)), + BlockLength: blockLength, + ChecksumLength: checksumLength, + } +} diff --git a/internal/rsyncdconfig/config.go b/internal/rsyncdconfig/config.go new file mode 100644 index 0000000..1336e3b --- /dev/null +++ b/internal/rsyncdconfig/config.go @@ -0,0 +1,57 @@ +package rsyncdconfig + +import ( + "os" + "path/filepath" + + "github.com/BurntSushi/toml" + "github.com/runpod/rsync/rsyncd" +) + +type SSHListener struct { + Address string `toml:"address"` + AuthorizedKeys string `toml:"authorized_keys"` +} + +type Listener struct { + HostKeyPath string `toml:"host_key_path"` + Rsyncd string `toml:"rsyncd"` + HTTPMonitoring string `toml:"http_monitoring"` + AnonSSH string `toml:"anon_ssh"` + AuthorizedSSH SSHListener `toml:"authorized_ssh"` +} + +type Config struct { + Listeners []Listener `toml:"listener"` + Modules []rsyncd.Module `toml:"module"` + DontNamespace bool `toml:"dont_namespace"` +} + +func FromString(input string) (*Config, error) { + var cfg Config + if _, err := toml.Decode(input, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func FromFile(path string) (*Config, error) { + input, err := os.ReadFile(path) + if err != nil { + return nil, err + } + return FromString(string(input)) +} + +func FromDefaultFiles() (*Config, string, error) { + configDir, err := os.UserConfigDir() + if err != nil { + return nil, "", err + } + fn := filepath.Join(configDir, "gokr-rsyncd.toml") + cfg, err := FromFile(fn) + if err != nil { + return nil, "", err + } + return cfg, fn, nil +} diff --git a/internal/rsyncdconfig/config_test.go b/internal/rsyncdconfig/config_test.go new file mode 100644 index 0000000..cafd608 --- /dev/null +++ b/internal/rsyncdconfig/config_test.go @@ -0,0 +1,54 @@ +package rsyncdconfig_test + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/rsyncd" +) + +func TestConfig(t *testing.T) { + cfg, err := rsyncdconfig.FromString(` +[[listener]] +rsyncd = "localhost:873" + +[[listener]] +http_monitoring = "localhost:8738" + +[[listener]] +anon_ssh = "localhost:22873" + +[[module]] +name = "interop" +path = "/non/existant/path" + +`) + if err != nil { + t.Fatal(err) + } + + if got, want := len(cfg.Listeners), 3; got != want { + t.Fatalf("unexpected number of listeners: got %d, want %d", got, want) + } + + { + want := []rsyncdconfig.Listener{ + {Rsyncd: "localhost:873"}, + {HTTPMonitoring: "localhost:8738"}, + {AnonSSH: "localhost:22873"}, + } + if diff := cmp.Diff(want, cfg.Listeners); diff != "" { + t.Fatalf("unexpected listener config: diff (-want +got):\n%s", diff) + } + } + + { + want := []rsyncd.Module{ + {Name: "interop", Path: "/non/existant/path"}, + } + if diff := cmp.Diff(want, cfg.Modules); diff != "" { + t.Fatalf("unexpected module config: diff (-want +got):\n%s", diff) + } + } +} diff --git a/internal/rsyncopts/popt.go b/internal/rsyncopts/popt.go new file mode 100644 index 0000000..0362e9e --- /dev/null +++ b/internal/rsyncopts/popt.go @@ -0,0 +1,273 @@ +package rsyncopts + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +type poptOption struct { + longName string + shortName string + argInfo int + arg any // depends on argInfo + val int // 0 means don't return, just update arg + // descrip string + // argDescrip string +} + +func (o *poptOption) name() string { + if o.longName == "" { + return "-" + o.shortName + } + return "--" + o.longName +} + +// see popt(3) +const ( + POPT_ARG_NONE = iota // int; No argument expected + POPT_ARG_STRING // char*; No type checking to be performed + POPT_ARG_INT // int; An integer argument is expected + POPT_ARG_LONG // long; A long integer is expected + POPT_ARG_INCLUDE_TABLE // nest another option table + POPT_ARG_CALLBACK // call a function + POPT_ARG_INTL_DOMAIN // + POPT_ARG_VAL // int; Integer value taken from val + POPT_ARG_FLOAT // float; A float argument is expected + POPT_ARG_DOUBLE // double; A double argument is expected + POPT_ARG_LONGLONG // long long; A long long integer is expected + POPT_ARG_MAINCALL = 16 + 11 + POPT_ARG_ARGV = 12 + POPT_ARG_SHORT = 13 + POPT_ARG_BITSET = 16 + 14 +) + +const POPT_ARG_MASK = 0x000000FF + +const ( + POPT_ARGFLAG_OR = 0x08000000 +) + +const ( + POPT_BIT_SET = POPT_ARG_VAL | POPT_ARGFLAG_OR +) + +type PoptError struct { + Errno int32 + Err error + DaemonMode bool +} + +func (pe *PoptError) Unwrap() error { return pe.Err } + +func (pe *PoptError) Error() string { + if pe.DaemonMode { + return pe.Err.Error() + " (in daemon mode)" + } + return pe.Err.Error() +} + +// TODO(later): turn these into sentinel error values +// which stringify like poptStrerror() +const ( + POPT_ERROR_NOARG = -10 // missing argument + POPT_ERROR_BADOPT = -11 // unknown option + POPT_ERROR_UNWANTEDARG = -12 // option does not take an argument + POPT_ERROR_OPTSTOODEEP = -13 // aliases nested too deeply + POPT_ERROR_BADQUOTE = -15 // error in parameter quoting + POPT_ERROR_ERRNO = -16 // errno set, use strerror(errno) + POPT_ERROR_BADNUMBER = -17 // invalid numeric value + POPT_ERROR_OVERFLOW = -18 // number too large or too small + POPT_ERROR_BADOPERATION = -19 // mutually exclusive logical operations requested + POPT_ERROR_NULLARG = -20 // opt->arg should not be NULL + POPT_ERROR_MALLOC = -21 // memory allocation failed + POPT_ERROR_BADCONFIG = -22 // config file failed sanity test +) + +type Context struct { + // state + table []poptOption + args []string + nextCharArg string + nextArg string + + // output + Options *Options + RemainingArgs []string +} + +func (pc *Context) findOption(longName, shortName string) *poptOption { + for idx, opt := range pc.table { + if longName != "" && opt.longName == longName { + return &pc.table[idx] + } + if shortName != "" && opt.shortName == shortName { + return &pc.table[idx] + } + } + return nil +} + +func (pc *Context) poptSaveInt(opt *poptOption, val int) bool { + intPtr := opt.arg.(*int) + if intPtr == nil { + return false + } + if opt.argInfo&POPT_ARGFLAG_OR != 0 { + *intPtr |= val + } else { + *intPtr = val + } + return true +} + +func (pc *Context) poptSaveArg(opt *poptOption, nextArg string) int32 { + argType := opt.argInfo & POPT_ARG_MASK + switch argType { + case POPT_ARG_INT: + i, err := strconv.ParseInt(nextArg, 0, 64) + if err != nil { + return POPT_ERROR_BADNUMBER + } + if i < math.MinInt32 || i > math.MaxInt32 { + return POPT_ERROR_OVERFLOW + } + pc.poptSaveInt(opt, int(i)) + return 0 + + case POPT_ARG_STRING: + stringPtr := opt.arg.(*string) + if stringPtr == nil { + return 0 + } + *stringPtr = nextArg + return 0 + } + + return POPT_ERROR_BADOPERATION +} + +func (pc *Context) poptGetNextOpt() (int32, error) { + var opt *poptOption + for { + var longArg string + if pc.nextCharArg == "" && len(pc.args) == 0 { + return -1, nil // done + } + if pc.nextCharArg == "" { + // process next long option + origOptString := pc.args[0] + pc.args = pc.args[1:] + if origOptString == "" { + return -1, &PoptError{ + Errno: POPT_ERROR_BADOPT, + Err: fmt.Errorf("unknown option: origOptString empty"), + } + } + if origOptString[0] != '-' || origOptString == "-" { + pc.RemainingArgs = append(pc.RemainingArgs, origOptString) + continue + } + before, after, found := strings.Cut(origOptString, "=") + if found { + longArg = after + } + // remove the one dash we ensured is present + before = strings.TrimPrefix(before, "-") + // a second dash is permitted + oneDash := false + if strings.HasPrefix(before, "-") { + before = strings.TrimPrefix(before, "-") + } else { + oneDash = true + } + opt = pc.findOption(before, "") + if opt == nil && !oneDash { + return -1, &PoptError{ + Errno: POPT_ERROR_BADOPT, + Err: fmt.Errorf("%s: unknown option", origOptString), + } + } + if opt == nil { + // try and parse it as a short option + pc.nextCharArg = origOptString[1:] + longArg = "" + } + } + if pc.nextCharArg != "" { + // process next short option + opt = pc.findOption("", pc.nextCharArg[:1]) + if opt == nil { + return -1, &PoptError{ + Errno: POPT_ERROR_BADOPT, + Err: fmt.Errorf("-%s: unknown option", pc.nextCharArg[:1]), + } + } + pc.nextCharArg = pc.nextCharArg[1:] + } + if opt == nil { + // neither long nor short? how can we end up here? + return -1, &PoptError{ + Errno: POPT_ERROR_BADOPT, + //lint:ignore ST1005 we need this punctuation for dramatic effect! + Err: fmt.Errorf("neither long nor short option found?!"), + } + } + argType := opt.argInfo & POPT_ARG_MASK + if argType == POPT_ARG_NONE || argType == POPT_ARG_VAL { + if longArg != "" || strings.HasPrefix(pc.nextCharArg, "=") { + return -1, &PoptError{ + Errno: POPT_ERROR_UNWANTEDARG, + Err: fmt.Errorf("option %s does not take an argument", opt.name()), + } + } + if opt.arg != nil { + val := 1 + if argType == POPT_ARG_VAL { + val = opt.val + } + if !pc.poptSaveInt(opt, val) { + return -1, &PoptError{ + Errno: POPT_ERROR_BADOPERATION, + Err: fmt.Errorf("poptSaveInt"), + } + } + } + } else { + nextArg := longArg + if longArg != "" { + } else if pc.nextCharArg != "" { + nextArg = strings.TrimPrefix(pc.nextCharArg, "=") + pc.nextCharArg = "" + } else { + if len(pc.args) == 0 { + return -1, &PoptError{ + Errno: POPT_ERROR_NOARG, + Err: fmt.Errorf("missing argument for option %s", opt.name()), + } + } + nextArg = pc.args[0] + pc.args = pc.args[1:] + } + pc.nextArg = nextArg + if opt.arg != nil { + if errno := pc.poptSaveArg(opt, nextArg); errno != 0 { + return -1, &PoptError{ + Errno: errno, + Err: fmt.Errorf("poptSaveArg errno %d", errno), + } + } + } + } + if opt.val != 0 && argType != POPT_ARG_VAL { + return int32(opt.val), nil + } + } +} + +func (pc *Context) poptGetOptArg() string { + ret := pc.nextArg + pc.nextArg = "" + return ret +} diff --git a/internal/rsyncopts/rsyncopts.go b/internal/rsyncopts/rsyncopts.go new file mode 100644 index 0000000..e55a648 --- /dev/null +++ b/internal/rsyncopts/rsyncopts.go @@ -0,0 +1,1597 @@ +// Package rsyncopts implements a parser for command-line options that +// implements a subset of popt(3) semantics; just enough to parse typical +// rsync(1) invocations without the advanced popt features like aliases +// or option prefix matching (not --del, only --delete). +// +// If we encounter arguments that rsync(1) parses differently compared to this +// package, then this package should be adjusted to match rsync(1). +package rsyncopts + +import ( + "errors" + "fmt" + "math" + "os" + "slices" + "strconv" + "strings" + "syscall" + "unicode" + + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/version" +) + +const ( + OPT_SERVER = 1000 + iota + OPT_DAEMON + OPT_SENDER + OPT_EXCLUDE + OPT_EXCLUDE_FROM + OPT_FILTER + OPT_COMPARE_DEST + OPT_COPY_DEST + OPT_LINK_DEST + OPT_HELP + OPT_INCLUDE + OPT_INCLUDE_FROM + OPT_MODIFY_WINDOW + OPT_MIN_SIZE + OPT_CHMOD + OPT_READ_BATCH + OPT_WRITE_BATCH + OPT_ONLY_WRITE_BATCH + OPT_MAX_SIZE + OPT_NO_D + OPT_APPEND + OPT_NO_ICONV + OPT_INFO + OPT_DEBUG + OPT_BLOCK_SIZE + OPT_USERMAP + OPT_GROUPMAP + OPT_CHOWN + OPT_BWLIMIT + OPT_STDERR + OPT_OLD_COMPRESS + OPT_NEW_COMPRESS + OPT_NO_COMPRESS + OPT_OLD_ARGS + OPT_STOP_AFTER + OPT_STOP_AT + OPT_REFUSED_BASE = 9000 +) + +type InfoLevel int + +const ( + INFO_BACKUP InfoLevel = iota + INFO_COPY + INFO_DEL + INFO_FLIST + INFO_MISC + INFO_MOUNT + INFO_NAME + INFO_NONREG + INFO_PROGRESS + INFO_REMOVE + INFO_SKIP + INFO_STATS + INFO_SYMSAFE + COUNT_INFO +) + +type DebugLevel int + +const ( + DEBUG_ACL DebugLevel = iota + DEBUG_BACKUP + DEBUG_BIND + DEBUG_CHDIR + DEBUG_CONNECT + DEBUG_CMD + DEBUG_DEL + DEBUG_DELTASUM + DEBUG_DUP + DEBUG_EXIT + DEBUG_FILTER + DEBUG_FLIST + DEBUG_FUZZY + DEBUG_GENR + DEBUG_HASH + DEBUG_HLINK + DEBUG_ICONV + DEBUG_IO + DEBUG_NSTR + DEBUG_OWN + DEBUG_PROTO + DEBUG_RECV + DEBUG_SEND + DEBUG_TIME + COUNT_DEBUG +) + +var tridgeDefaults = Options{ + msgs2stderr: 2, // Default: send errors to stderr for local & remote-shell transfers + output_motd: 1, + human_readable: 1, + allow_inc_recurse: 1, + xfer_dirs: -1, + relative_paths: -1, + implied_dirs: 1, + max_delete: math.MinInt32, + whole_file: -1, + do_compression_level: math.MinInt32, + rsync_path: "rsync", + default_af_hint: syscall.AF_INET6, + blocking_io: -1, + protocol_version: 27, +} + +// NewOptions returns an Options struct with all options initialized to their +// default values. Note that ParseArguments will set some options (that default +// to -1) based on the encountered command-line flags and built-in rules. +func NewOptions(osenv *rsyncos.Env) *Options { + opts := tridgeDefaults // copy + opts.osenv = osenv + opts.table = func() []poptOption { + return opts.tridgeTable() + } + return &opts +} + +var gokrazyDefaults = Options{ + msgs2stderr: 2, // Default: send errors to stderr for local & remote-shell transfers + output_motd: 1, + human_readable: 1, + // TODO: if/when allow_inc_recurse gets implemented, + // default to 1 to match tridge rsync + allow_inc_recurse: 0, + xfer_dirs: -1, + relative_paths: -1, + implied_dirs: 1, + max_delete: math.MinInt32, + whole_file: -1, + do_compression_level: math.MinInt32, + rsync_path: "rsync", + default_af_hint: syscall.AF_INET6, + blocking_io: -1, + protocol_version: 27, +} + +// NewOptions returns an Options struct with all options initialized to their +// default values. Note that ParseArguments will set some options (that default +// to -1) based on the encountered command-line flags and built-in rules. +func NewOptionsWithGokrazyDefaults(osenv *rsyncos.Env) *Options { + opts := gokrazyDefaults // copy + opts.osenv = osenv + opts.table = func() []poptOption { + return opts.gokrazyTable() + } + return &opts +} + +// GokrazyClientOptions contains additional command-line flags, prefixed with +// gokr. (like --gokr.dont_restrict) to not clash with rsync flag names. +type GokrazyClientOptions struct { + DontRestrict int +} + +func (o *GokrazyClientOptions) table() []poptOption { + return []poptOption{ + /* longName, shortName, argInfo, arg, val */ + {"gokr.dont_restrict", "", POPT_ARG_NONE, &o.DontRestrict, 0}, + } +} + +// GokrazyDaemonOptions contains additional command-line flags, prefixed with +// gokr. (like --gokr.modulemap) to not clash with rsync flag names. +type GokrazyDaemonOptions struct { + Config string + Listen string + MonitoringListen string + AnonSSHListen string + ModuleMap string +} + +func (o *GokrazyDaemonOptions) table() []poptOption { + return []poptOption{ + /* longName, shortName, argInfo, arg, val */ + {"gokr.config", "", POPT_ARG_STRING, &o.Config, 0}, + {"gokr.listen", "", POPT_ARG_STRING, &o.Listen, 0}, + {"gokr.monitoring_listen", "", POPT_ARG_STRING, &o.MonitoringListen, 0}, + {"gokr.anonssh_listen", "", POPT_ARG_STRING, &o.AnonSSHListen, 0}, + {"gokr.modulemap", "", POPT_ARG_STRING, &o.ModuleMap, 0}, + } +} + +type Options struct { + osenv *rsyncos.Env + table func() []poptOption + + GokrazyClient GokrazyClientOptions + GokrazyDaemon GokrazyDaemonOptions + + // not directly referenced in the table, but used in the special case code. + do_compression int + info [COUNT_INFO]uint16 + debug [COUNT_DEBUG]uint16 + local_server int + filterRules []string + + // order matches long_options order + verbose int + msgs2stderr int + quiet int + output_motd int + do_stats int + human_readable int + dry_run int + recurse int + allow_inc_recurse int + xfer_dirs int + preserve_perms int + preserve_executability int + preserve_acls int + preserve_xattrs int + preserve_mtimes int + preserve_atimes int + open_noatime int + preserve_crtimes int + omit_dir_times int + omit_link_times int + modify_window int + am_root int // 0 = normal, 1 = root, 2 = --super, -1 = --fake-super + preserve_uid int + preserve_gid int + preserve_devices int + copy_devices int + write_devices int + preserve_specials int + preserve_links int + copy_links int + copy_unsafe_links int + safe_symlinks int + munge_symlinks int + copy_dirlinks int + keep_dirlinks int + preserve_hard_links int + relative_paths int + implied_dirs int + ignore_times int + size_only int + one_file_system int + update_only int + ignore_non_existing int + ignore_existing int + max_size_arg string + min_size_arg string + max_alloc_arg string + sparse_files int + preallocate_files int + inplace int + append_mode int + delete_during int + delete_mode int + delete_before int + delete_after int + delete_excluded int + missing_args int // 0 = FERROR_XFER, 1 = ignore, 2 = delete + remove_source_files int + force_delete int + ignore_errors int + max_delete int + cvs_exclude int + // If 1, send the whole file as literal data rather than trying to create an + // incremental diff. + // If -1, then look at whether we're local or remote and go by that. + // See also disable_deltas_p() + whole_file int + always_checksum int + checksum_choice string + fuzzy_basis int + compress_choice string + skip_compress string + do_compression_level int + do_progress int + keep_partial int + partial_dir string + delay_updates int + prune_empty_dirs int + logfile_name string + logfile_format string + stdout_format string + itemize_changes int + bwlimit_arg string + bwlimit int + make_backups int + backup_dir string + backup_suffix string + list_only int + batch_name string + files_from string + eol_nulls int + old_style_args int // intentionally set to 0; unsupported + protect_args int // intentionally set to 0; currently unsupported + trust_sender int + numeric_ids int + io_timeout int + connect_timeout int + do_fsync int + shell_cmd string + rsync_path string + tmpdir string + iconv_opt string + default_af_hint int + allow_8bit_chars int + mkpath_dest_arg int + use_qsort int + copy_as string + bind_address string // numeric IPv4 or IPv6, or a hostname + rsync_port int + sockopts string + password_file string + early_input_file string + blocking_io int + outbuf_mode string + protocol_version int + checksum_seed int + am_server int + am_sender int + am_daemon int + + daemon_bwlimit int + config_file string + daemon_opt int + no_detach int +} + +type priority int + +const ( + DEFAULT_PRIORITY priority = iota + HELP_PRIORITY + USER_PRIORITY + LIMIT_PRIORITY +) + +const ( + W_CLI = 1 << iota + W_SRV + W_SND + W_REC +) + +type output struct { + name string + where int + help string +} + +var infoWords = [...]output{ + {"BACKUP", W_REC, "Mention files backed up"}, + {"COPY", W_REC, "Mention files copied locally on the receiving side"}, + {"DEL", W_REC, "Mention deletions on the receiving side"}, + {"FLIST", W_CLI, "Mention file-list receiving/sending (levels 1-2)"}, + {"MISC", W_SND | W_REC, "Mention miscellaneous information (levels 1-2)"}, + {"MOUNT", W_SND | W_REC, "Mention mounts that were found or skipped"}, + {"NAME", W_SND | W_REC, "Mention 1) updated file/dir names, 2) unchanged names"}, + {"NONREG", W_REC, "Mention skipped non-regular files (default 1, 0 disables)"}, + {"PROGRESS", W_CLI, "Mention 1) per-file progress or 2) total transfer progress"}, + {"REMOVE", W_SND, "Mention files removed on the sending side"}, + {"SKIP", W_REC, "Mention files skipped due to transfer overrides (levels 1-2)"}, + {"STATS", W_CLI | W_SRV, "Mention statistics at end of run (levels 1-3)"}, + {"SYMSAFE", W_SND | W_REC, "Mention symlinks that are unsafe"}, +} + +var debugWords = [...]output{ + {"ACL", W_SND | W_REC, "Debug extra ACL info"}, + {"BACKUP", W_REC, "Debug backup actions (levels 1-2)"}, + {"CHDIR", W_CLI | W_SRV, "Debug when the current directory changes"}, + {"BIND", W_CLI, "Debug socket bind actions"}, + {"CONNECT", W_CLI, "Debug connection events (levels 1-2)"}, + {"CMD", W_CLI, "Debug commands+options that are issued (levels 1-2)"}, + {"DEL", W_REC, "Debug delete actions (levels 1-3)"}, + {"DELTASUM", W_SND | W_REC, "Debug delta-transfer checksumming (levels 1-4)"}, + {"DUP", W_REC, "Debug weeding of duplicate names"}, + {"EXIT", W_CLI | W_SRV, "Debug exit events (levels 1-3)"}, + {"FILTER", W_SND | W_REC, "Debug filter actions (levels 1-3)"}, + {"FLIST", W_SND | W_REC, "Debug file-list operations (levels 1-4)"}, + {"FUZZY", W_REC, "Debug fuzzy scoring (levels 1-2)"}, + {"GENR", W_REC, "Debug generator functions"}, + {"HASH", W_SND | W_REC, "Debug hashtable code"}, + {"HLINK", W_SND | W_REC, "Debug hard-link actions (levels 1-3)"}, + {"ICONV", W_CLI | W_SRV, "Debug iconv character conversions (levels 1-2)"}, + {"IO", W_CLI | W_SRV, "Debug I/O routines (levels 1-4)"}, + {"NSTR", W_CLI | W_SRV, "Debug negotiation strings"}, + {"OWN", W_REC, "Debug ownership changes in users & groups (levels 1-2)"}, + {"PROTO", W_CLI | W_SRV, "Debug protocol information"}, + {"RECV", W_REC, "Debug receiver functions"}, + {"SEND", W_SND, "Debug sender functions"}, + {"TIME", W_REC, "Debug setting of modified times (levels 1-2)"}, +} + +func parseOutputWords(osenv *rsyncos.Env, words []output, levels []uint16, str string, prio priority) error { +Level: + for s := range strings.SplitSeq(str, ",") { + if strings.TrimSpace(s) == "" { + continue + } + trimmed := strings.TrimRightFunc(s, unicode.IsNumber) + lev := 1 + if len(trimmed) < len(s) { + var err error + lev, err = strconv.Atoi(s[len(trimmed):]) + if err != nil { + return err + } + } + trimmed = strings.ToLower(trimmed) + all := false + switch trimmed { + case "help": + osenv.Logf("TODO: print --info/--debug help and exit") + os.Exit(0) + case "none": + lev = 0 + case "all": + all = true + } + for j := range words { + word := words[j] + if strings.ToLower(word.name) == trimmed || all { + levels[j] = uint16(lev) + if !all { + continue Level + } + } + } + if !all { + return fmt.Errorf("unknown --info/--debug item: %q", trimmed) + } + } + return nil +} + +func (o *Options) setOutputVerbosity(prio priority) error { + debugVerbosity := [...]string{ + "", + "", + "BIND,CMD,CONNECT,DEL,DELTASUM,DUP,FILTER,FLIST,ICONV", + "ACL,BACKUP,CONNECT2,DELTASUM2,DEL2,EXIT,FILTER2,FLIST2,FUZZY,GENR,OWN,RECV,SEND,TIME", + "CMD2,DELTASUM3,DEL3,EXIT2,FLIST3,ICONV2,OWN2,PROTO,TIME2", + "CHDIR,DELTASUM4,FLIST4,FUZZY2,HASH,HLINK", + } + infoVerbosity := [...]string{ + "NONREG", + "COPY,DEL,FLIST,MISC,NAME,STATS,SYMSAFE", + "BACKUP,MISC2,MOUNT,NAME2,REMOVE,SKIP", + } + for j := 0; j <= o.verbose; j++ { + if j < len(infoVerbosity) { + if err := parseOutputWords(o.osenv, infoWords[:], o.info[:], infoVerbosity[j], prio); err != nil { + return err + } + } + if j < len(debugVerbosity) { + if err := parseOutputWords(o.osenv, debugWords[:], o.debug[:], debugVerbosity[j], prio); err != nil { + return err + } + } + } + return nil +} + +func (o *Options) DaemonHelp() string { + return version.Read() + ` +gokrazy/rsync is a native Go rsync implementation. +It recognizes all command-line flags that the original rsync supports, +but might not implement all functionality (and instead error out). + +See the rsync(1) man page for more details on rsync. +For your convenience, here is the rsync --daemon --help output: + + --daemon run as an rsync daemon + --address=ADDRESS bind to the specified address + --bwlimit=RATE limit socket I/O bandwidth + --config=FILE specify alternate rsyncd.conf file + --dparam=OVERRIDE, -M override global daemon config parameter + --no-detach do not detach from the parent + --port=PORT listen on alternate port number + --log-file=FILE override the "log file" setting + --log-file-format=FMT override the "log format" setting + --sockopts=OPTIONS specify custom TCP options + --verbose, -v increase verbosity + --ipv4, -4 prefer IPv4 + --ipv6, -6 prefer IPv6 + --help, -h show this help (when used with --daemon) + +In addition, the following gokrazy-specific flags are supported: + + --gokr.config path to a config file (if unspecified, + os.UserConfigDir()/gokr-rsyncd.toml is used) + --gokr.listen [host]:port listen address for the rsync daemon protocol + --gokr.monitoring_listen optional [host]:port listen address for a HTTP debug interface + --gokr.anonssh_listen optional [host]:port listen address for + the rsync daemon protocol via anonymous SSH + --gokr.modulemap = pairs for quick setup of the server, + without a config file + +See https://github.com/runpod/rsync for updates, bug reports, and answers +` +} + +func (o *Options) Help() string { + return version.Read() + ` + +gokrazy/rsync is a native Go rsync implementation. +It recognizes all command-line flags that the original rsync supports, +but might not implement all functionality (and instead error out). + +See the rsync(1) man page for more details on rsync. +For your convenience, here is the rsync --help output: + + rsync is a file transfer program capable of efficient remote update + via a fast differencing algorithm. + + Usage: rsync [OPTION]... SRC [SRC]... DEST + or rsync [OPTION]... SRC [SRC]... [USER@]HOST:DEST + or rsync [OPTION]... SRC [SRC]... [USER@]HOST::DEST + or rsync [OPTION]... SRC [SRC]... rsync://[USER@]HOST[:PORT]/DEST + or rsync [OPTION]... [USER@]HOST:SRC [DEST] + or rsync [OPTION]... [USER@]HOST::SRC [DEST] + or rsync [OPTION]... rsync://[USER@]HOST[:PORT]/SRC [DEST] + The ':' usages connect via remote shell, while '::' & 'rsync://' usages connect + to an rsync daemon, and require SRC or DEST to start with a module name. + + Options: + + --verbose, -v increase verbosity + --info=FLAGS fine-grained informational verbosity + --debug=FLAGS fine-grained debug verbosity + --stderr=e|a|c change stderr output mode (default: errors) + --quiet, -q suppress non-error messages + --no-motd suppress daemon-mode MOTD + --checksum, -c skip based on checksum, not mod-time & size + --archive, -a archive mode is -rlptgoD (no -A,-X,-U,-N,-H) + --no-OPTION turn off an implied OPTION (e.g. --no-D) + --recursive, -r recurse into directories + --relative, -R use relative path names + --no-implied-dirs don't send implied dirs with --relative + --backup, -b make backups (see --suffix & --backup-dir) + --backup-dir=DIR make backups into hierarchy based in DIR + --suffix=SUFFIX backup suffix (default ~ w/o --backup-dir) + --update, -u skip files that are newer on the receiver + --inplace update destination files in-place + --append append data onto shorter files + --append-verify --append w/old data in file checksum + --dirs, -d transfer directories without recursing + --old-dirs, --old-d works like --dirs when talking to old rsync + --mkpath create destination's missing path components + --links, -l copy symlinks as symlinks + --copy-links, -L transform symlink into referent file/dir + --copy-unsafe-links only "unsafe" symlinks are transformed + --safe-links ignore symlinks that point outside the tree + --munge-links munge symlinks to make them safe & unusable + --copy-dirlinks, -k transform symlink to dir into referent dir + --keep-dirlinks, -K treat symlinked dir on receiver as dir + --hard-links, -H preserve hard links + --perms, -p preserve permissions + --executability, -E preserve executability + --chmod=CHMOD affect file and/or directory permissions + --acls, -A preserve ACLs (implies --perms) + --xattrs, -X preserve extended attributes + --owner, -o preserve owner (super-user only) + --group, -g preserve group + --devices preserve device files (super-user only) + --copy-devices copy device contents as a regular file + --write-devices write to devices as files (implies --inplace) + --specials preserve special files + -D same as --devices --specials + --times, -t preserve modification times + --atimes, -U preserve access (use) times + --open-noatime avoid changing the atime on opened files + --crtimes, -N preserve create times (newness) + --omit-dir-times, -O omit directories from --times + --omit-link-times, -J omit symlinks from --times + --super receiver attempts super-user activities + --fake-super store/recover privileged attrs using xattrs + --sparse, -S turn sequences of nulls into sparse blocks + --preallocate allocate dest files before writing them + --dry-run, -n perform a trial run with no changes made + --whole-file, -W copy files whole (w/o delta-xfer algorithm) + --checksum-choice=STR choose the checksum algorithm (aka --cc) + --one-file-system, -x don't cross filesystem boundaries + --block-size=SIZE, -B force a fixed checksum block-size + --rsh=COMMAND, -e specify the remote shell to use + --rsync-path=PROGRAM specify the rsync to run on remote machine + --existing skip creating new files on receiver + --ignore-existing skip updating files that exist on receiver + --remove-source-files sender removes synchronized files (non-dir) + --del an alias for --delete-during + --delete delete extraneous files from dest dirs + --delete-before receiver deletes before xfer, not during + --delete-during receiver deletes during the transfer + --delete-delay find deletions during, delete after + --delete-after receiver deletes after transfer, not during + --delete-excluded also delete excluded files from dest dirs + --ignore-missing-args ignore missing source args without error + --delete-missing-args delete missing source args from destination + --ignore-errors delete even if there are I/O errors + --force force deletion of dirs even if not empty + --max-delete=NUM don't delete more than NUM files + --max-size=SIZE don't transfer any file larger than SIZE + --min-size=SIZE don't transfer any file smaller than SIZE + --max-alloc=SIZE change a limit relating to memory alloc + --partial keep partially transferred files + --partial-dir=DIR put a partially transferred file into DIR + --delay-updates put all updated files into place at end + --prune-empty-dirs, -m prune empty directory chains from file-list + --numeric-ids don't map uid/gid values by user/group name + --usermap=STRING custom username mapping + --groupmap=STRING custom groupname mapping + --chown=USER:GROUP simple username/groupname mapping + --timeout=SECONDS set I/O timeout in seconds + --contimeout=SECONDS set daemon connection timeout in seconds + --ignore-times, -I don't skip files that match size and time + --size-only skip files that match in size + --modify-window=NUM, -@ set the accuracy for mod-time comparisons + --temp-dir=DIR, -T create temporary files in directory DIR + --fuzzy, -y find similar file for basis if no dest file + --compare-dest=DIR also compare destination files relative to DIR + --copy-dest=DIR ... and include copies of unchanged files + --link-dest=DIR hardlink to files in DIR when unchanged + --compress, -z compress file data during the transfer + --compress-choice=STR choose the compression algorithm (aka --zc) + --compress-level=NUM explicitly set compression level (aka --zl) + --skip-compress=LIST skip compressing files with suffix in LIST + --cvs-exclude, -C auto-ignore files in the same way CVS does + --filter=RULE, -f add a file-filtering RULE + -F same as --filter='dir-merge /.rsync-filter' + repeated: --filter='- .rsync-filter' + --exclude=PATTERN exclude files matching PATTERN + --exclude-from=FILE read exclude patterns from FILE + --include=PATTERN don't exclude files matching PATTERN + --include-from=FILE read include patterns from FILE + --files-from=FILE read list of source-file names from FILE + --from0, -0 all *-from/filter files are delimited by 0s + --old-args disable the modern arg-protection idiom + --secluded-args, -s use the protocol to safely send the args + --trust-sender trust the remote sender's file list + --copy-as=USER[:GROUP] specify user & optional group for the copy + --address=ADDRESS bind address for outgoing socket to daemon + --port=PORT specify double-colon alternate port number + --sockopts=OPTIONS specify custom TCP options + --blocking-io use blocking I/O for the remote shell + --outbuf=N|L|B set out buffering to None, Line, or Block + --stats give some file-transfer stats + --8-bit-output, -8 leave high-bit chars unescaped in output + --human-readable, -h output numbers in a human-readable format + --progress show progress during transfer + -P same as --partial --progress + --itemize-changes, -i output a change-summary for all updates + --remote-option=OPT, -M send OPTION to the remote side only + --out-format=FORMAT output updates using the specified FORMAT + --log-file=FILE log what we're doing to the specified FILE + --log-file-format=FMT log updates using the specified FMT + --password-file=FILE read daemon-access password from FILE + --early-input=FILE use FILE for daemon's early exec input + --list-only list the files instead of copying them + --bwlimit=RATE limit socket I/O bandwidth + --stop-after=MINS Stop rsync after MINS minutes have elapsed + --stop-at=y-m-dTh:m Stop rsync at the specified point in time + --fsync fsync every written file + --write-batch=FILE write a batched update to FILE + --only-write-batch=FILE like --write-batch but w/o updating dest + --read-batch=FILE read a batched update from FILE + --protocol=NUM force an older protocol version to be used + --iconv=CONVERT_SPEC request charset conversion of filenames + --checksum-seed=NUM set block/file checksum seed (advanced) + --ipv4, -4 prefer IPv4 + --ipv6, -6 prefer IPv6 + --version, -V print the version + other info and exit + --help, -h (*) show this help (* -h is help only on its own) + + Use "rsync --daemon --help" to see the daemon-mode command-line options. + +In addition, the following gokrazy-specific flags are supported: + + --gokr.dont_restrict do not restrict file system access to source/dest + where available (e.g. with Landlock on Linux) + +See https://github.com/runpod/rsync for updates, bug reports, and answers +` +} + +func (o *Options) ShellCommand() string { return o.shell_cmd } +func (o *Options) UpdateOnly() bool { return o.update_only != 0 } +func (o *Options) DryRun() bool { return o.dry_run != 0 } +func (o *Options) PreserveLinks() bool { return o.preserve_links != 0 } +func (o *Options) PreserveUid() bool { return o.preserve_uid != 0 } +func (o *Options) PreserveGid() bool { return o.preserve_gid != 0 } +func (o *Options) PreserveDevices() bool { return o.preserve_devices != 0 } +func (o *Options) PreserveMTimes() bool { return o.preserve_mtimes != 0 } +func (o *Options) PreservePerms() bool { return o.preserve_perms != 0 } +func (o *Options) PreserveSpecials() bool { return o.preserve_specials != 0 } +func (o *Options) PreserveHardLinks() bool { return o.preserve_hard_links != 0 } +func (o *Options) DoFsync() bool { return o.do_fsync != 0 } +func (o *Options) Recurse() bool { return o.recurse != 0 } +func (o *Options) Verbose() bool { return o.verbose != 0 } +func (o *Options) DeleteMode() bool { return o.delete_mode != 0 } +func (o *Options) Sender() bool { return o.am_sender != 0 } +func (o *Options) SetSender() { o.am_sender = 1 } +func (o *Options) LocalServer() bool { return o.local_server != 0 } +func (o *Options) SetLocalServer() { o.local_server = 1 } +func (o *Options) Server() bool { return o.am_server != 0 } +func (o *Options) Daemon() bool { return o.am_daemon != 0 } +func (o *Options) ConnectTimeoutSeconds() int { return o.connect_timeout } +func (o *Options) AlwaysChecksum() bool { return o.always_checksum != 0 } +func (o *Options) IgnoreTimes() bool { return o.ignore_times != 0 } +func (o *Options) KeepPartial() bool { return o.keep_partial != 0 } +func (o *Options) OutputMOTD() bool { return o.output_motd != 0 } +func (o *Options) RsyncPort() int { return o.rsync_port } +func (o *Options) XferDirs() int { return o.xfer_dirs } +func (o *Options) FilterRules() []string { return o.filterRules } +func (o *Options) Progress() bool { + return o.info[INFO_PROGRESS] > 0 +} + +func (o *Options) InfoGTE(flag InfoLevel, lvl uint16) bool { + return o.info[int(flag)] >= lvl +} + +func (o *Options) InfoEQ(flag InfoLevel, lvl uint16) bool { + return o.info[int(flag)] == lvl +} + +func (o *Options) DebugGTE(flag DebugLevel, lvl uint16) bool { + return o.debug[int(flag)] >= lvl +} + +func (o *Options) DebugEQ(flag DebugLevel, lvl uint16) bool { + return o.debug[int(flag)] == lvl +} + +func (o *Options) daemonTable() []poptOption { + return []poptOption{ + /* longName, shortName, argInfo, arg, val */ + {"address", "", POPT_ARG_STRING, &o.bind_address, 0}, + {"bwlimit", "", POPT_ARG_INT, &o.daemon_bwlimit, 0}, + {"config", "", POPT_ARG_STRING, &o.config_file, 0}, + {"daemon", "", POPT_ARG_NONE, &o.daemon_opt, 0}, + {"dparam", "M", POPT_ARG_STRING, nil, 'M'}, + {"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET}, + {"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6}, + {"detach", "", POPT_ARG_VAL, &o.no_detach, 0}, + {"no-detach", "", POPT_ARG_VAL, &o.no_detach, 1}, + {"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0}, + {"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0}, + {"port", "", POPT_ARG_INT, &o.rsync_port, 0}, + {"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0}, + {"protocol", "", POPT_ARG_INT, &o.protocol_version, 0}, + {"server", "", POPT_ARG_NONE, &o.am_server, 0}, + {"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0}, + {"verbose", "v", POPT_ARG_NONE, nil, 'v'}, + {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0}, + {"no-v", "", POPT_ARG_VAL, &o.verbose, 0}, + {"help", "h", POPT_ARG_NONE, nil, 'h'}, + } +} + +func (o *Options) gokrazyTable() []poptOption { + // Commented out options are not yet implemented. + return []poptOption{ + /* longName, shortName, argInfo, arg, val */ + {"help", "", POPT_ARG_NONE, nil, OPT_HELP}, + {"version", "V", POPT_ARG_NONE, nil, 'V'}, + {"verbose", "v", POPT_ARG_NONE, nil, 'v'}, + {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0}, + {"no-v", "", POPT_ARG_VAL, &o.verbose, 0}, + {"info", "", POPT_ARG_STRING, nil, OPT_INFO}, + {"debug", "", POPT_ARG_STRING, nil, OPT_DEBUG}, + //{"stderr", "", POPT_ARG_STRING, nil, OPT_STDERR}, + //{"msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 1}, + //{"no-msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 0}, + //{"quiet", "q", POPT_ARG_NONE, nil, 'q'}, + {"motd", "", POPT_ARG_VAL, &o.output_motd, 1}, + {"no-motd", "", POPT_ARG_VAL, &o.output_motd, 0}, + //{"stats", "", POPT_ARG_NONE, &o.do_stats, 0}, + //{"human-readable", "h", POPT_ARG_NONE, nil, 'h'}, + //{"no-human-readable", "", POPT_ARG_VAL, &o.human_readable, 0}, + //{"no-h", "", POPT_ARG_VAL, &o.human_readable, 0}, + {"dry-run", "n", POPT_ARG_NONE, &o.dry_run, 0}, + {"archive", "a", POPT_ARG_NONE, nil, 'a'}, + {"recursive", "r", POPT_ARG_VAL, &o.recurse, 2}, + {"no-recursive", "", POPT_ARG_VAL, &o.recurse, 0}, + {"no-r", "", POPT_ARG_VAL, &o.recurse, 0}, + //{"inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1}, + //{"no-inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0}, + //{"i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1}, + //{"no-i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0}, + {"dirs", "d", POPT_ARG_VAL, &o.xfer_dirs, 2}, + {"no-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 0}, + {"no-d", "", POPT_ARG_VAL, &o.xfer_dirs, 0}, + //{"old-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 4}, + //{"old-d", "", POPT_ARG_VAL, &o.xfer_dirs, 4}, + {"perms", "p", POPT_ARG_VAL, &o.preserve_perms, 1}, + {"no-perms", "", POPT_ARG_VAL, &o.preserve_perms, 0}, + {"no-p", "", POPT_ARG_VAL, &o.preserve_perms, 0}, + //{"executability", "E", POPT_ARG_NONE, &o.preserve_executability, 0}, + //{"acls", "A", POPT_ARG_NONE, nil, 'A'}, + //{"no-acls", "", POPT_ARG_VAL, &o.preserve_acls, 0}, + //{"no-A", "", POPT_ARG_VAL, &o.preserve_acls, 0}, + //{"xattrs", "X", POPT_ARG_NONE, nil, 'X'}, + //{"no-xattrs", "", POPT_ARG_VAL, &o.preserve_xattrs, 0}, + //{"no-X", "", POPT_ARG_VAL, &o.preserve_xattrs, 0}, + {"times", "t", POPT_ARG_VAL, &o.preserve_mtimes, 1}, + {"no-times", "", POPT_ARG_VAL, &o.preserve_mtimes, 0}, + {"no-t", "", POPT_ARG_VAL, &o.preserve_mtimes, 0}, + //{"atimes", "U", POPT_ARG_NONE, nil, 'U'}, + //{"no-atimes", "", POPT_ARG_VAL, &o.preserve_atimes, 0}, + //{"no-U", "", POPT_ARG_VAL, &o.preserve_atimes, 0}, + //{"open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 1}, + //{"no-open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 0}, + //{"crtimes", "N", POPT_ARG_NONE, &o.preserve_crtimes, 1}, // refused + //{"no-crtimes", "", POPT_ARG_VAL, &o.preserve_crtimes, 0}, + //{"no-N", "", POPT_ARG_VAL, &o.preserve_crtimes, 0}, + //{"omit-dir-times", "O", POPT_ARG_VAL, &o.omit_dir_times, 1}, + //{"no-omit-dir-times", "", POPT_ARG_VAL, &o.omit_dir_times, 0}, + //{"no-O", "", POPT_ARG_VAL, &o.omit_dir_times, 0}, + //{"omit-link-times", "J", POPT_ARG_VAL, &o.omit_link_times, 1}, + //{"no-omit-link-times", "", POPT_ARG_VAL, &o.omit_link_times, 0}, + //{"no-J", "", POPT_ARG_VAL, &o.omit_link_times, 0}, + //{"modify-window", "@", POPT_ARG_INT, &o.modify_window, OPT_MODIFY_WINDOW}, + //{"super", "", POPT_ARG_VAL, &o.am_root, 2}, + //{"no-super", "", POPT_ARG_VAL, &o.am_root, 0}, + //{"fake-super", "", POPT_ARG_VAL, &o.am_root, -1}, + {"owner", "o", POPT_ARG_VAL, &o.preserve_uid, 1}, + {"no-owner", "", POPT_ARG_VAL, &o.preserve_uid, 0}, + {"no-o", "", POPT_ARG_VAL, &o.preserve_uid, 0}, + {"group", "g", POPT_ARG_VAL, &o.preserve_gid, 1}, + {"no-group", "", POPT_ARG_VAL, &o.preserve_gid, 0}, + {"no-g", "", POPT_ARG_VAL, &o.preserve_gid, 0}, + {"", "D", POPT_ARG_NONE, nil, 'D'}, + {"no-D", "", POPT_ARG_NONE, nil, OPT_NO_D}, + {"devices", "", POPT_ARG_VAL, &o.preserve_devices, 1}, + {"no-devices", "", POPT_ARG_VAL, &o.preserve_devices, 0}, + //{"copy-devices", "", POPT_ARG_NONE, &o.copy_devices, 0}, + //{"write-devices", "", POPT_ARG_VAL, &o.write_devices, 1}, + //{"no-write-devices", "", POPT_ARG_VAL, &o.write_devices, 0}, + {"specials", "", POPT_ARG_VAL, &o.preserve_specials, 1}, + {"no-specials", "", POPT_ARG_VAL, &o.preserve_specials, 0}, + {"links", "l", POPT_ARG_VAL, &o.preserve_links, 1}, + {"no-links", "", POPT_ARG_VAL, &o.preserve_links, 0}, + {"no-l", "", POPT_ARG_VAL, &o.preserve_links, 0}, + //{"copy-links", "L", POPT_ARG_NONE, &o.copy_links, 0}, + //{"copy-unsafe-links", "", POPT_ARG_NONE, &o.copy_unsafe_links, 0}, + //{"safe-links", "", POPT_ARG_NONE, &o.safe_symlinks, 0}, + //{"munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 1}, + //{"no-munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 0}, + //{"copy-dirlinks", "k", POPT_ARG_NONE, &o.copy_dirlinks, 0}, + //{"keep-dirlinks", "K", POPT_ARG_NONE, &o.keep_dirlinks, 0}, + {"hard-links", "H", POPT_ARG_NONE, nil, 'H'}, + {"no-hard-links", "", POPT_ARG_VAL, &o.preserve_hard_links, 0}, + {"no-H", "", POPT_ARG_VAL, &o.preserve_hard_links, 0}, + //{"relative", "R", POPT_ARG_VAL, &o.relative_paths, 1}, + //{"no-relative", "", POPT_ARG_VAL, &o.relative_paths, 0}, + //{"no-R", "", POPT_ARG_VAL, &o.relative_paths, 0}, + //{"implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 1}, + //{"no-implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 0}, + //{"i-d", "", POPT_ARG_VAL, &o.implied_dirs, 1}, + //{"no-i-d", "", POPT_ARG_VAL, &o.implied_dirs, 0}, + //{"chmod", "", POPT_ARG_STRING, nil, OPT_CHMOD}, + {"ignore-times", "I", POPT_ARG_NONE, &o.ignore_times, 0}, + //{"size-only", "", POPT_ARG_NONE, &o.size_only, 0}, + //{"one-file-system", "x", POPT_ARG_NONE, nil, 'x'}, + //{"no-one-file-system", "", POPT_ARG_VAL, &o.one_file_system, 0}, + //{"no-x", "", POPT_ARG_VAL, &o.one_file_system, 0}, + {"update", "u", POPT_ARG_NONE, &o.update_only, 0}, + //{"existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0}, + //{"ignore-non-existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0}, + //{"ignore-existing", "", POPT_ARG_NONE, &o.ignore_existing, 0}, + //{"max-size", "", POPT_ARG_STRING, &o.max_size_arg, OPT_MAX_SIZE}, + //{"min-size", "", POPT_ARG_STRING, &o.min_size_arg, OPT_MIN_SIZE}, + //{"max-alloc", "", POPT_ARG_STRING, &o.max_alloc_arg, 0}, + //{"sparse", "S", POPT_ARG_VAL, &o.sparse_files, 1}, + //{"no-sparse", "", POPT_ARG_VAL, &o.sparse_files, 0}, + //{"no-S", "", POPT_ARG_VAL, &o.sparse_files, 0}, + //{"preallocate", "", POPT_ARG_NONE, &o.preallocate_files, 0}, + //{"inplace", "", POPT_ARG_VAL, &o.inplace, 1}, + //{"no-inplace", "", POPT_ARG_VAL, &o.inplace, 0}, + //{"append", "", POPT_ARG_NONE, nil, OPT_APPEND}, + //{"append-verify", "", POPT_ARG_VAL, &o.append_mode, 2}, + //{"no-append", "", POPT_ARG_VAL, &o.append_mode, 0}, + //{"del", "", POPT_ARG_NONE, &o.delete_during, 0}, + {"delete", "", POPT_ARG_NONE, &o.delete_mode, 0}, + //{"delete-before", "", POPT_ARG_NONE, &o.delete_before, 0}, + //{"delete-during", "", POPT_ARG_VAL, &o.delete_during, 1}, + //{"delete-delay", "", POPT_ARG_VAL, &o.delete_during, 2}, + //{"delete-after", "", POPT_ARG_NONE, &o.delete_after, 0}, + //{"delete-excluded", "", POPT_ARG_NONE, &o.delete_excluded, 0}, + //{"delete-missing-args", "", POPT_BIT_SET, &o.missing_args, 2}, + //{"ignore-missing-args", "", POPT_BIT_SET, &o.missing_args, 1}, + //{"remove-sent-files", "", POPT_ARG_VAL, &o.remove_source_files, 2}, /* deprecated */ + //{"remove-source-files", "", POPT_ARG_VAL, &o.remove_source_files, 1}, + //{"force", "", POPT_ARG_VAL, &o.force_delete, 1}, + //{"no-force", "", POPT_ARG_VAL, &o.force_delete, 0}, + //{"ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 1}, + //{"no-ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 0}, + //{"max-delete", "", POPT_ARG_INT, &o.max_delete, 0}, + //{"", "F", POPT_ARG_NONE, nil, 'F'}, + {"filter", "f", POPT_ARG_STRING, nil, OPT_FILTER}, + {"exclude", "", POPT_ARG_STRING, nil, OPT_EXCLUDE}, + {"include", "", POPT_ARG_STRING, nil, OPT_INCLUDE}, + //{"exclude-from", "", POPT_ARG_STRING, nil, OPT_EXCLUDE_FROM}, + //{"include-from", "", POPT_ARG_STRING, nil, OPT_INCLUDE_FROM}, + //{"cvs-exclude", "C", POPT_ARG_NONE, &o.cvs_exclude, 0}, + //{"whole-file", "W", POPT_ARG_VAL, &o.whole_file, 1}, + //{"no-whole-file", "", POPT_ARG_VAL, &o.whole_file, 0}, + //{"no-W", "", POPT_ARG_VAL, &o.whole_file, 0}, + {"checksum", "c", POPT_ARG_VAL, &o.always_checksum, 1}, + {"no-checksum", "", POPT_ARG_VAL, &o.always_checksum, 0}, + {"no-c", "", POPT_ARG_VAL, &o.always_checksum, 0}, + //{"checksum-choice", "", POPT_ARG_STRING, &o.checksum_choice, 0}, + //{"cc", "", POPT_ARG_STRING, &o.checksum_choice, 0}, + //{"block-size", "B", POPT_ARG_STRING, nil, OPT_BLOCK_SIZE}, + //{"compare-dest", "", POPT_ARG_STRING, nil, OPT_COMPARE_DEST}, + //{"copy-dest", "", POPT_ARG_STRING, nil, OPT_COPY_DEST}, + //{"link-dest", "", POPT_ARG_STRING, nil, OPT_LINK_DEST}, + //{"fuzzy", "y", POPT_ARG_NONE, nil, 'y'}, + //{"no-fuzzy", "", POPT_ARG_VAL, &o.fuzzy_basis, 0}, + //{"no-y", "", POPT_ARG_VAL, &o.fuzzy_basis, 0}, + + // Regarding compression support, see: + // https://github.com/runpod/rsync/issues/35#issuecomment-2988582190 + // + //{"compress", "z", POPT_ARG_NONE, nil, 'z'}, + //{"old-compress", "", POPT_ARG_NONE, nil, OPT_OLD_COMPRESS}, + //{"new-compress", "", POPT_ARG_NONE, nil, OPT_NEW_COMPRESS}, + //{"no-compress", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS}, + //{"no-z", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS}, + //{"compress-choice", "", POPT_ARG_STRING, &o.compress_choice, 0}, + //{"zc", "", POPT_ARG_STRING, &o.compress_choice, 0}, + //{"skip-compress", "", POPT_ARG_STRING, &o.skip_compress, 0}, + //{"compress-level", "", POPT_ARG_INT, &o.do_compression_level, 0}, + //{"zl", "", POPT_ARG_INT, &o.do_compression_level, 0}, + + //{"", "P", POPT_ARG_NONE, nil, 'P'}, + {"progress", "", POPT_ARG_VAL, &o.do_progress, 1}, + {"no-progress", "", POPT_ARG_VAL, &o.do_progress, 0}, + {"partial", "", POPT_ARG_VAL, &o.keep_partial, 1}, + {"no-partial", "", POPT_ARG_VAL, &o.keep_partial, 0}, + //{"partial-dir", "", POPT_ARG_STRING, &o.partial_dir, 0}, + //{"delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 1}, + //{"no-delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 0}, + //{"prune-empty-dirs", "m", POPT_ARG_VAL, &o.prune_empty_dirs, 1}, + //{"no-prune-empty-dirs", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0}, + //{"no-m", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0}, + //{"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0}, + //{"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0}, + //{"out-format", "", POPT_ARG_STRING, &o.stdout_format, 0}, + //{"log-format", "", POPT_ARG_STRING, &o.stdout_format, 0}, /* DEPRECATED */ + //{"itemize-changes", "i", POPT_ARG_NONE, nil, 'i'}, + //{"no-itemize-changes", "", POPT_ARG_VAL, &o.itemize_changes, 0}, + //{"no-i", "", POPT_ARG_VAL, &o.itemize_changes, 0}, + //{"bwlimit", "", POPT_ARG_STRING, &o.bwlimit_arg, OPT_BWLIMIT}, + //{"no-bwlimit", "", POPT_ARG_VAL, &o.bwlimit, 0}, + //{"backup", "b", POPT_ARG_VAL, &o.make_backups, 1}, + //{"no-backup", "", POPT_ARG_VAL, &o.make_backups, 0}, + //{"backup-dir", "", POPT_ARG_STRING, &o.backup_dir, 0}, + //{"suffix", "", POPT_ARG_STRING, &o.backup_suffix, 0}, + {"list-only", "", POPT_ARG_VAL, &o.list_only, 2}, + //{"read-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_READ_BATCH}, + //{"write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_WRITE_BATCH}, + //{"only-write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_ONLY_WRITE_BATCH}, + //{"files-from", "", POPT_ARG_STRING, &o.files_from, 0}, + //{"from0", "0", POPT_ARG_VAL, &o.eol_nulls, 1}, + //{"no-from0", "", POPT_ARG_VAL, &o.eol_nulls, 0}, + //{"old-args", "", POPT_ARG_NONE, nil, OPT_OLD_ARGS}, + //{"no-old-args", "", POPT_ARG_VAL, &o.old_style_args, 0}, + //{"secluded-args", "s", POPT_ARG_VAL, &o.protect_args, 1}, + //{"no-secluded-args", "", POPT_ARG_VAL, &o.protect_args, 0}, + //{"protect-args", "", POPT_ARG_VAL, &o.protect_args, 1}, + //{"no-protect-args", "", POPT_ARG_VAL, &o.protect_args, 0}, + //{"no-s", "", POPT_ARG_VAL, &o.protect_args, 0}, + //{"trust-sender", "", POPT_ARG_VAL, &o.trust_sender, 1}, + //{"numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 1}, + //{"no-numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 0}, + //{"usermap", "", POPT_ARG_STRING, nil, OPT_USERMAP}, + //{"groupmap", "", POPT_ARG_STRING, nil, OPT_GROUPMAP}, + //{"chown", "", POPT_ARG_STRING, nil, OPT_CHOWN}, + //{"timeout", "", POPT_ARG_INT, &o.io_timeout, 0}, + //{"no-timeout", "", POPT_ARG_VAL, &o.io_timeout, 0}, + {"contimeout", "", POPT_ARG_INT, &o.connect_timeout, 0}, + {"no-contimeout", "", POPT_ARG_VAL, &o.connect_timeout, 0}, + {"fsync", "", POPT_ARG_NONE, &o.do_fsync, 0}, + //{"stop-after", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER}, + //{"time-limit", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER}, /* earlier stop-after name */ + //{"stop-at", "", POPT_ARG_STRING, nil, OPT_STOP_AT}, + {"rsh", "e", POPT_ARG_STRING, &o.shell_cmd, 0}, + //{"rsync-path", "", POPT_ARG_STRING, &o.rsync_path, 0}, + //{"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0}, + //{"iconv", "", POPT_ARG_STRING, &o.iconv_opt, 0}, + //{"no-iconv", "", POPT_ARG_NONE, nil, OPT_NO_ICONV}, + //{"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET}, + //{"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6}, + //{"8-bit-output", "8", POPT_ARG_VAL, &o.allow_8bit_chars, 1}, + //{"no-8-bit-output", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0}, + //{"no-8", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0}, + //{"mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 1}, + //{"no-mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 0}, + //{"qsort", "", POPT_ARG_NONE, &o.use_qsort, 0}, + //{"copy-as", "", POPT_ARG_STRING, &o.copy_as, 0}, + //{"address", "", POPT_ARG_STRING, &o.bind_address, 0}, + {"port", "", POPT_ARG_INT, &o.rsync_port, 0}, + //{"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0}, + //{"password-file", "", POPT_ARG_STRING, &o.password_file, 0}, + //{"early-input", "", POPT_ARG_STRING, &o.early_input_file, 0}, + //{"blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 1}, + //{"no-blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 0}, + //{"outbuf", "", POPT_ARG_STRING, &o.outbuf_mode, 0}, + //{"remote-option", "M", POPT_ARG_STRING, nil, 'M'}, + //{"protocol", "", POPT_ARG_INT, &o.protocol_version, 0}, + //{"checksum-seed", "", POPT_ARG_INT, &o.checksum_seed, 0}, + {"server", "", POPT_ARG_NONE, nil, OPT_SERVER}, + {"sender", "", POPT_ARG_NONE, nil, OPT_SENDER}, + /* All the following options switch us into daemon-mode option-parsing. */ + {"config", "", POPT_ARG_STRING, nil, OPT_DAEMON}, + {"daemon", "", POPT_ARG_NONE, nil, OPT_DAEMON}, + {"dparam", "", POPT_ARG_STRING, nil, OPT_DAEMON}, + {"detach", "", POPT_ARG_NONE, nil, OPT_DAEMON}, + {"no-detach", "", POPT_ARG_NONE, nil, OPT_DAEMON}, + } +} + +func (o *Options) tridgeTable() []poptOption { + return []poptOption{ + /* longName, shortName, argInfo, arg, val */ + {"help", "", POPT_ARG_NONE, nil, OPT_HELP}, + {"version", "V", POPT_ARG_NONE, nil, 'V'}, + {"verbose", "v", POPT_ARG_NONE, nil, 'v'}, + {"no-verbose", "", POPT_ARG_VAL, &o.verbose, 0}, + {"no-v", "", POPT_ARG_VAL, &o.verbose, 0}, + {"info", "", POPT_ARG_STRING, nil, OPT_INFO}, + {"debug", "", POPT_ARG_STRING, nil, OPT_DEBUG}, + {"stderr", "", POPT_ARG_STRING, nil, OPT_STDERR}, + {"msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 1}, + {"no-msgs2stderr", "", POPT_ARG_VAL, &o.msgs2stderr, 0}, + {"quiet", "q", POPT_ARG_NONE, nil, 'q'}, + {"motd", "", POPT_ARG_VAL, &o.output_motd, 1}, + {"no-motd", "", POPT_ARG_VAL, &o.output_motd, 0}, + {"stats", "", POPT_ARG_NONE, &o.do_stats, 0}, + {"human-readable", "h", POPT_ARG_NONE, nil, 'h'}, + {"no-human-readable", "", POPT_ARG_VAL, &o.human_readable, 0}, + {"no-h", "", POPT_ARG_VAL, &o.human_readable, 0}, + {"dry-run", "n", POPT_ARG_NONE, &o.dry_run, 0}, + {"archive", "a", POPT_ARG_NONE, nil, 'a'}, + {"recursive", "r", POPT_ARG_VAL, &o.recurse, 2}, + {"no-recursive", "", POPT_ARG_VAL, &o.recurse, 0}, + {"no-r", "", POPT_ARG_VAL, &o.recurse, 0}, + {"inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1}, + {"no-inc-recursive", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0}, + {"i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 1}, + {"no-i-r", "", POPT_ARG_VAL, &o.allow_inc_recurse, 0}, + {"dirs", "d", POPT_ARG_VAL, &o.xfer_dirs, 2}, + {"no-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 0}, + {"no-d", "", POPT_ARG_VAL, &o.xfer_dirs, 0}, + {"old-dirs", "", POPT_ARG_VAL, &o.xfer_dirs, 4}, + {"old-d", "", POPT_ARG_VAL, &o.xfer_dirs, 4}, + {"perms", "p", POPT_ARG_VAL, &o.preserve_perms, 1}, + {"no-perms", "", POPT_ARG_VAL, &o.preserve_perms, 0}, + {"no-p", "", POPT_ARG_VAL, &o.preserve_perms, 0}, + {"executability", "E", POPT_ARG_NONE, &o.preserve_executability, 0}, + {"acls", "A", POPT_ARG_NONE, nil, 'A'}, + {"no-acls", "", POPT_ARG_VAL, &o.preserve_acls, 0}, + {"no-A", "", POPT_ARG_VAL, &o.preserve_acls, 0}, + {"xattrs", "X", POPT_ARG_NONE, nil, 'X'}, + {"no-xattrs", "", POPT_ARG_VAL, &o.preserve_xattrs, 0}, + {"no-X", "", POPT_ARG_VAL, &o.preserve_xattrs, 0}, + {"times", "t", POPT_ARG_VAL, &o.preserve_mtimes, 1}, + {"no-times", "", POPT_ARG_VAL, &o.preserve_mtimes, 0}, + {"no-t", "", POPT_ARG_VAL, &o.preserve_mtimes, 0}, + {"atimes", "U", POPT_ARG_NONE, nil, 'U'}, + {"no-atimes", "", POPT_ARG_VAL, &o.preserve_atimes, 0}, + {"no-U", "", POPT_ARG_VAL, &o.preserve_atimes, 0}, + {"open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 1}, + {"no-open-noatime", "", POPT_ARG_VAL, &o.open_noatime, 0}, + {"crtimes", "N", POPT_ARG_NONE, &o.preserve_crtimes, 1}, // refused + {"no-crtimes", "", POPT_ARG_VAL, &o.preserve_crtimes, 0}, + {"no-N", "", POPT_ARG_VAL, &o.preserve_crtimes, 0}, + {"omit-dir-times", "O", POPT_ARG_VAL, &o.omit_dir_times, 1}, + {"no-omit-dir-times", "", POPT_ARG_VAL, &o.omit_dir_times, 0}, + {"no-O", "", POPT_ARG_VAL, &o.omit_dir_times, 0}, + {"omit-link-times", "J", POPT_ARG_VAL, &o.omit_link_times, 1}, + {"no-omit-link-times", "", POPT_ARG_VAL, &o.omit_link_times, 0}, + {"no-J", "", POPT_ARG_VAL, &o.omit_link_times, 0}, + {"modify-window", "@", POPT_ARG_INT, &o.modify_window, OPT_MODIFY_WINDOW}, + {"super", "", POPT_ARG_VAL, &o.am_root, 2}, + {"no-super", "", POPT_ARG_VAL, &o.am_root, 0}, + {"fake-super", "", POPT_ARG_VAL, &o.am_root, -1}, + {"owner", "o", POPT_ARG_VAL, &o.preserve_uid, 1}, + {"no-owner", "", POPT_ARG_VAL, &o.preserve_uid, 0}, + {"no-o", "", POPT_ARG_VAL, &o.preserve_uid, 0}, + {"group", "g", POPT_ARG_VAL, &o.preserve_gid, 1}, + {"no-group", "", POPT_ARG_VAL, &o.preserve_gid, 0}, + {"no-g", "", POPT_ARG_VAL, &o.preserve_gid, 0}, + {"", "D", POPT_ARG_NONE, nil, 'D'}, + {"no-D", "", POPT_ARG_NONE, nil, OPT_NO_D}, + {"devices", "", POPT_ARG_VAL, &o.preserve_devices, 1}, + {"no-devices", "", POPT_ARG_VAL, &o.preserve_devices, 0}, + {"copy-devices", "", POPT_ARG_NONE, &o.copy_devices, 0}, + {"write-devices", "", POPT_ARG_VAL, &o.write_devices, 1}, + {"no-write-devices", "", POPT_ARG_VAL, &o.write_devices, 0}, + {"specials", "", POPT_ARG_VAL, &o.preserve_specials, 1}, + {"no-specials", "", POPT_ARG_VAL, &o.preserve_specials, 0}, + {"links", "l", POPT_ARG_VAL, &o.preserve_links, 1}, + {"no-links", "", POPT_ARG_VAL, &o.preserve_links, 0}, + {"no-l", "", POPT_ARG_VAL, &o.preserve_links, 0}, + {"copy-links", "L", POPT_ARG_NONE, &o.copy_links, 0}, + {"copy-unsafe-links", "", POPT_ARG_NONE, &o.copy_unsafe_links, 0}, + {"safe-links", "", POPT_ARG_NONE, &o.safe_symlinks, 0}, + {"munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 1}, + {"no-munge-links", "", POPT_ARG_VAL, &o.munge_symlinks, 0}, + {"copy-dirlinks", "k", POPT_ARG_NONE, &o.copy_dirlinks, 0}, + {"keep-dirlinks", "K", POPT_ARG_NONE, &o.keep_dirlinks, 0}, + {"hard-links", "H", POPT_ARG_NONE, nil, 'H'}, + {"no-hard-links", "", POPT_ARG_VAL, &o.preserve_hard_links, 0}, + {"no-H", "", POPT_ARG_VAL, &o.preserve_hard_links, 0}, + {"relative", "R", POPT_ARG_VAL, &o.relative_paths, 1}, + {"no-relative", "", POPT_ARG_VAL, &o.relative_paths, 0}, + {"no-R", "", POPT_ARG_VAL, &o.relative_paths, 0}, + {"implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 1}, + {"no-implied-dirs", "", POPT_ARG_VAL, &o.implied_dirs, 0}, + {"i-d", "", POPT_ARG_VAL, &o.implied_dirs, 1}, + {"no-i-d", "", POPT_ARG_VAL, &o.implied_dirs, 0}, + {"chmod", "", POPT_ARG_STRING, nil, OPT_CHMOD}, + {"ignore-times", "I", POPT_ARG_NONE, &o.ignore_times, 0}, + {"size-only", "", POPT_ARG_NONE, &o.size_only, 0}, + {"one-file-system", "x", POPT_ARG_NONE, nil, 'x'}, + {"no-one-file-system", "", POPT_ARG_VAL, &o.one_file_system, 0}, + {"no-x", "", POPT_ARG_VAL, &o.one_file_system, 0}, + {"update", "u", POPT_ARG_NONE, &o.update_only, 0}, + {"existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0}, + {"ignore-non-existing", "", POPT_ARG_NONE, &o.ignore_non_existing, 0}, + {"ignore-existing", "", POPT_ARG_NONE, &o.ignore_existing, 0}, + {"max-size", "", POPT_ARG_STRING, &o.max_size_arg, OPT_MAX_SIZE}, + {"min-size", "", POPT_ARG_STRING, &o.min_size_arg, OPT_MIN_SIZE}, + {"max-alloc", "", POPT_ARG_STRING, &o.max_alloc_arg, 0}, + {"sparse", "S", POPT_ARG_VAL, &o.sparse_files, 1}, + {"no-sparse", "", POPT_ARG_VAL, &o.sparse_files, 0}, + {"no-S", "", POPT_ARG_VAL, &o.sparse_files, 0}, + {"preallocate", "", POPT_ARG_NONE, &o.preallocate_files, 0}, + {"inplace", "", POPT_ARG_VAL, &o.inplace, 1}, + {"no-inplace", "", POPT_ARG_VAL, &o.inplace, 0}, + {"append", "", POPT_ARG_NONE, nil, OPT_APPEND}, + {"append-verify", "", POPT_ARG_VAL, &o.append_mode, 2}, + {"no-append", "", POPT_ARG_VAL, &o.append_mode, 0}, + {"del", "", POPT_ARG_NONE, &o.delete_during, 0}, + {"delete", "", POPT_ARG_NONE, &o.delete_mode, 0}, + {"delete-before", "", POPT_ARG_NONE, &o.delete_before, 0}, + {"delete-during", "", POPT_ARG_VAL, &o.delete_during, 1}, + {"delete-delay", "", POPT_ARG_VAL, &o.delete_during, 2}, + {"delete-after", "", POPT_ARG_NONE, &o.delete_after, 0}, + {"delete-excluded", "", POPT_ARG_NONE, &o.delete_excluded, 0}, + {"delete-missing-args", "", POPT_BIT_SET, &o.missing_args, 2}, + {"ignore-missing-args", "", POPT_BIT_SET, &o.missing_args, 1}, + {"remove-sent-files", "", POPT_ARG_VAL, &o.remove_source_files, 2}, /* deprecated */ + {"remove-source-files", "", POPT_ARG_VAL, &o.remove_source_files, 1}, + {"force", "", POPT_ARG_VAL, &o.force_delete, 1}, + {"no-force", "", POPT_ARG_VAL, &o.force_delete, 0}, + {"ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 1}, + {"no-ignore-errors", "", POPT_ARG_VAL, &o.ignore_errors, 0}, + {"max-delete", "", POPT_ARG_INT, &o.max_delete, 0}, + {"", "F", POPT_ARG_NONE, nil, 'F'}, + {"filter", "f", POPT_ARG_STRING, nil, OPT_FILTER}, + {"exclude", "", POPT_ARG_STRING, nil, OPT_EXCLUDE}, + {"include", "", POPT_ARG_STRING, nil, OPT_INCLUDE}, + {"exclude-from", "", POPT_ARG_STRING, nil, OPT_EXCLUDE_FROM}, + {"include-from", "", POPT_ARG_STRING, nil, OPT_INCLUDE_FROM}, + {"cvs-exclude", "C", POPT_ARG_NONE, &o.cvs_exclude, 0}, + {"whole-file", "W", POPT_ARG_VAL, &o.whole_file, 1}, + {"no-whole-file", "", POPT_ARG_VAL, &o.whole_file, 0}, + {"no-W", "", POPT_ARG_VAL, &o.whole_file, 0}, + {"checksum", "c", POPT_ARG_VAL, &o.always_checksum, 1}, + {"no-checksum", "", POPT_ARG_VAL, &o.always_checksum, 0}, + {"no-c", "", POPT_ARG_VAL, &o.always_checksum, 0}, + {"checksum-choice", "", POPT_ARG_STRING, &o.checksum_choice, 0}, + {"cc", "", POPT_ARG_STRING, &o.checksum_choice, 0}, + {"block-size", "B", POPT_ARG_STRING, nil, OPT_BLOCK_SIZE}, + {"compare-dest", "", POPT_ARG_STRING, nil, OPT_COMPARE_DEST}, + {"copy-dest", "", POPT_ARG_STRING, nil, OPT_COPY_DEST}, + {"link-dest", "", POPT_ARG_STRING, nil, OPT_LINK_DEST}, + {"fuzzy", "y", POPT_ARG_NONE, nil, 'y'}, + {"no-fuzzy", "", POPT_ARG_VAL, &o.fuzzy_basis, 0}, + {"no-y", "", POPT_ARG_VAL, &o.fuzzy_basis, 0}, + {"compress", "z", POPT_ARG_NONE, nil, 'z'}, + {"old-compress", "", POPT_ARG_NONE, nil, OPT_OLD_COMPRESS}, + {"new-compress", "", POPT_ARG_NONE, nil, OPT_NEW_COMPRESS}, + {"no-compress", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS}, + {"no-z", "", POPT_ARG_NONE, nil, OPT_NO_COMPRESS}, + {"compress-choice", "", POPT_ARG_STRING, &o.compress_choice, 0}, + {"zc", "", POPT_ARG_STRING, &o.compress_choice, 0}, + {"skip-compress", "", POPT_ARG_STRING, &o.skip_compress, 0}, + {"compress-level", "", POPT_ARG_INT, &o.do_compression_level, 0}, + {"zl", "", POPT_ARG_INT, &o.do_compression_level, 0}, + {"", "P", POPT_ARG_NONE, nil, 'P'}, + {"progress", "", POPT_ARG_VAL, &o.do_progress, 1}, + {"no-progress", "", POPT_ARG_VAL, &o.do_progress, 0}, + {"partial", "", POPT_ARG_VAL, &o.keep_partial, 1}, + {"no-partial", "", POPT_ARG_VAL, &o.keep_partial, 0}, + {"partial-dir", "", POPT_ARG_STRING, &o.partial_dir, 0}, + {"delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 1}, + {"no-delay-updates", "", POPT_ARG_VAL, &o.delay_updates, 0}, + {"prune-empty-dirs", "m", POPT_ARG_VAL, &o.prune_empty_dirs, 1}, + {"no-prune-empty-dirs", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0}, + {"no-m", "", POPT_ARG_VAL, &o.prune_empty_dirs, 0}, + {"log-file", "", POPT_ARG_STRING, &o.logfile_name, 0}, + {"log-file-format", "", POPT_ARG_STRING, &o.logfile_format, 0}, + {"out-format", "", POPT_ARG_STRING, &o.stdout_format, 0}, + {"log-format", "", POPT_ARG_STRING, &o.stdout_format, 0}, /* DEPRECATED */ + {"itemize-changes", "i", POPT_ARG_NONE, nil, 'i'}, + {"no-itemize-changes", "", POPT_ARG_VAL, &o.itemize_changes, 0}, + {"no-i", "", POPT_ARG_VAL, &o.itemize_changes, 0}, + {"bwlimit", "", POPT_ARG_STRING, &o.bwlimit_arg, OPT_BWLIMIT}, + {"no-bwlimit", "", POPT_ARG_VAL, &o.bwlimit, 0}, + {"backup", "b", POPT_ARG_VAL, &o.make_backups, 1}, + {"no-backup", "", POPT_ARG_VAL, &o.make_backups, 0}, + {"backup-dir", "", POPT_ARG_STRING, &o.backup_dir, 0}, + {"suffix", "", POPT_ARG_STRING, &o.backup_suffix, 0}, + {"list-only", "", POPT_ARG_VAL, &o.list_only, 2}, + {"read-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_READ_BATCH}, + {"write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_WRITE_BATCH}, + {"only-write-batch", "", POPT_ARG_STRING, &o.batch_name, OPT_ONLY_WRITE_BATCH}, + {"files-from", "", POPT_ARG_STRING, &o.files_from, 0}, + {"from0", "0", POPT_ARG_VAL, &o.eol_nulls, 1}, + {"no-from0", "", POPT_ARG_VAL, &o.eol_nulls, 0}, + {"old-args", "", POPT_ARG_NONE, nil, OPT_OLD_ARGS}, + {"no-old-args", "", POPT_ARG_VAL, &o.old_style_args, 0}, + {"secluded-args", "s", POPT_ARG_VAL, &o.protect_args, 1}, + {"no-secluded-args", "", POPT_ARG_VAL, &o.protect_args, 0}, + {"protect-args", "", POPT_ARG_VAL, &o.protect_args, 1}, + {"no-protect-args", "", POPT_ARG_VAL, &o.protect_args, 0}, + {"no-s", "", POPT_ARG_VAL, &o.protect_args, 0}, + {"trust-sender", "", POPT_ARG_VAL, &o.trust_sender, 1}, + {"numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 1}, + {"no-numeric-ids", "", POPT_ARG_VAL, &o.numeric_ids, 0}, + {"usermap", "", POPT_ARG_STRING, nil, OPT_USERMAP}, + {"groupmap", "", POPT_ARG_STRING, nil, OPT_GROUPMAP}, + {"chown", "", POPT_ARG_STRING, nil, OPT_CHOWN}, + {"timeout", "", POPT_ARG_INT, &o.io_timeout, 0}, + {"no-timeout", "", POPT_ARG_VAL, &o.io_timeout, 0}, + {"contimeout", "", POPT_ARG_INT, &o.connect_timeout, 0}, + {"no-contimeout", "", POPT_ARG_VAL, &o.connect_timeout, 0}, + {"fsync", "", POPT_ARG_NONE, &o.do_fsync, 0}, + {"stop-after", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER}, + {"time-limit", "", POPT_ARG_STRING, nil, OPT_STOP_AFTER}, /* earlier stop-after name */ + {"stop-at", "", POPT_ARG_STRING, nil, OPT_STOP_AT}, + {"rsh", "e", POPT_ARG_STRING, &o.shell_cmd, 0}, + {"rsync-path", "", POPT_ARG_STRING, &o.rsync_path, 0}, + {"temp-dir", "T", POPT_ARG_STRING, &o.tmpdir, 0}, + {"iconv", "", POPT_ARG_STRING, &o.iconv_opt, 0}, + {"no-iconv", "", POPT_ARG_NONE, nil, OPT_NO_ICONV}, + {"ipv4", "4", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET}, + {"ipv6", "6", POPT_ARG_VAL, &o.default_af_hint, syscall.AF_INET6}, + {"8-bit-output", "8", POPT_ARG_VAL, &o.allow_8bit_chars, 1}, + {"no-8-bit-output", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0}, + {"no-8", "", POPT_ARG_VAL, &o.allow_8bit_chars, 0}, + {"mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 1}, + {"no-mkpath", "", POPT_ARG_VAL, &o.mkpath_dest_arg, 0}, + {"qsort", "", POPT_ARG_NONE, &o.use_qsort, 0}, + {"copy-as", "", POPT_ARG_STRING, &o.copy_as, 0}, + {"address", "", POPT_ARG_STRING, &o.bind_address, 0}, + {"port", "", POPT_ARG_INT, &o.rsync_port, 0}, + {"sockopts", "", POPT_ARG_STRING, &o.sockopts, 0}, + {"password-file", "", POPT_ARG_STRING, &o.password_file, 0}, + {"early-input", "", POPT_ARG_STRING, &o.early_input_file, 0}, + {"blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 1}, + {"no-blocking-io", "", POPT_ARG_VAL, &o.blocking_io, 0}, + {"outbuf", "", POPT_ARG_STRING, &o.outbuf_mode, 0}, + {"remote-option", "M", POPT_ARG_STRING, nil, 'M'}, + {"protocol", "", POPT_ARG_INT, &o.protocol_version, 0}, + {"checksum-seed", "", POPT_ARG_INT, &o.checksum_seed, 0}, + {"server", "", POPT_ARG_NONE, nil, OPT_SERVER}, + {"sender", "", POPT_ARG_NONE, nil, OPT_SENDER}, + /* All the following options switch us into daemon-mode option-parsing. */ + {"config", "", POPT_ARG_STRING, nil, OPT_DAEMON}, + {"daemon", "", POPT_ARG_NONE, nil, OPT_DAEMON}, + {"dparam", "", POPT_ARG_STRING, nil, OPT_DAEMON}, + {"detach", "", POPT_ARG_NONE, nil, OPT_DAEMON}, + {"no-detach", "", POPT_ARG_NONE, nil, OPT_DAEMON}, + } +} + +var errNotYetImplemented = errors.New("option not yet implemented in gokrazy/rsync") + +func NewContext(opts *Options) *Context { + table := opts.table() + table = slices.Concat(opts.GokrazyClient.table(), table) + return &Context{ + Options: opts, + table: table, + } +} + +// rsync/options.c:parse_arguments +func (pc *Context) ParseArguments(osenv *rsyncos.Env, args []string) error { + // NOTE: We do not implement support for refusing options per rsyncd.conf + // here, as we have our own configuration file. + + version_opt_cnt := 0 + + pc.args = args + opts := pc.Options + + for { + opt, err := pc.poptGetNextOpt() + if err != nil { + return err + } + if opt == -1 { + break // done + } + // Most options are handled by poptGetNextOpt, only special cases + // are returned and handled here. + switch opt { + case 'V': + version_opt_cnt++ + + case OPT_SERVER: + opts.am_server = 1 + + case OPT_SENDER: + if opts.am_server == 0 { + return fmt.Errorf("--sender only allowed with --server") + } + opts.am_sender = 1 + + case OPT_DAEMON: + // Parse the whole command-line using the daemon options table. + table := opts.daemonTable() + table = slices.Concat(opts.GokrazyDaemon.table(), table) + pc := Context{ + Options: opts, + table: table, + args: args, + } + + for { + opt, err := pc.poptGetNextOpt() + if err != nil { + err.(*PoptError).DaemonMode = true + return err + } + if opt == -1 { + break // done + } + // Most options are handled by poptGetNextOpt, only special cases + // are returned and handled here. + switch opt { + case 'h': + fmt.Println(opts.DaemonHelp()) // tridge rsync prints help to stdout + os.Exit(0) // exit with code 0 for compatibility with tridge rsync + case 'M': + return errNotYetImplemented + + case 'v': + opts.verbose++ + + default: + return fmt.Errorf("unhandled special case opt: %v", opt) + } + } + + opts.am_daemon = 1 + + return nil + + case OPT_FILTER: + opts.filterRules = append(opts.filterRules, pc.poptGetOptArg()) + case OPT_EXCLUDE: + opts.filterRules = append(opts.filterRules, "- "+pc.poptGetOptArg()) + case OPT_INCLUDE: + opts.filterRules = append(opts.filterRules, "+ "+pc.poptGetOptArg()) + + case OPT_INCLUDE_FROM, + OPT_EXCLUDE_FROM: + return errNotYetImplemented + + case 'a': + if opts.recurse == 0 { + opts.recurse = 1 + } + opts.preserve_links = 1 + opts.preserve_perms = 1 + opts.preserve_mtimes = 1 + opts.preserve_gid = 1 + opts.preserve_uid = 1 + opts.preserve_devices = 1 + opts.preserve_specials = 1 + + case 'D': + opts.preserve_devices = 1 + opts.preserve_specials = 1 + + case OPT_NO_D: + opts.preserve_devices = 0 + opts.preserve_specials = 0 + + case 'h': + opts.human_readable++ + + case 'H': + opts.preserve_hard_links = 1 + + case 'i': + opts.itemize_changes++ + + case 'U': + opts.preserve_atimes++ + if opts.preserve_atimes > 1 { + opts.open_noatime = 1 + } + + case 'v': + opts.verbose++ + + case 'y': + return errNotYetImplemented + + case 'q': + opts.quiet++ + + case 'x': + opts.one_file_system++ + + case 'F': + return errNotYetImplemented + + case 'P': + opts.do_progress = 1 + opts.keep_partial = 1 + + case 'z': + opts.do_compression++ + + case OPT_OLD_COMPRESS: + opts.compress_choice = "zlib" + + case OPT_NEW_COMPRESS: + opts.compress_choice = "zlibx" + + case OPT_NO_COMPRESS: + opts.do_compression = 0 + opts.compress_choice = "" + + case OPT_OLD_ARGS: + return errNotYetImplemented + + case 'M': // --remote-option + return errNotYetImplemented + + case OPT_WRITE_BATCH, + OPT_ONLY_WRITE_BATCH, + OPT_READ_BATCH: + return errNotYetImplemented + + case OPT_BLOCK_SIZE: + return errNotYetImplemented + + case OPT_MAX_SIZE, // (needs parse_size_arg) + OPT_MIN_SIZE, + OPT_BWLIMIT: + return errNotYetImplemented + + case OPT_APPEND: + return errNotYetImplemented + + case OPT_LINK_DEST, + OPT_COPY_DEST, + OPT_COMPARE_DEST: + return errNotYetImplemented + + case OPT_CHMOD: // (needs parse_chmod): + return errNotYetImplemented + + case OPT_INFO: + parseOutputWords(osenv, infoWords[:], opts.info[:], pc.poptGetOptArg(), USER_PRIORITY) + + case OPT_DEBUG: + parseOutputWords(osenv, debugWords[:], opts.debug[:], pc.poptGetOptArg(), USER_PRIORITY) + + case OPT_USERMAP, + OPT_GROUPMAP, + OPT_CHOWN: + return errNotYetImplemented + + case OPT_HELP: + fmt.Println(opts.Help()) // tridge rsync prints help to stdout + os.Exit(0) // exit with code 0 for compatibility with tridge rsync + + case 'A': + return fmt.Errorf("ACLs are not supported by gokrazy/rsync") + + case 'X': + opts.preserve_xattrs++ + + case OPT_STOP_AFTER, + OPT_STOP_AT, + OPT_STDERR: + return errNotYetImplemented + + default: + return fmt.Errorf("unhandled special case opt: %v", opt) + } + } + + // rsync/options.c line 1973 and following set option defaults based on + // other options + + if version_opt_cnt > 0 { + fmt.Println(version.Read()) + os.Exit(0) + } + + if opts.human_readable > 1 && len(args) == 1 /* && !am_server */ { + fmt.Println(opts.Help()) // tridge rsync prints help to stdout + os.Exit(0) // exit with code 0 for compatibility with tridge rsync + } + + if err := opts.setOutputVerbosity(DEFAULT_PRIORITY); err != nil { + // TODO: plumb error + fmt.Println(err.Error()) + os.Exit(1) + } + + if opts.recurse != 0 { + opts.xfer_dirs = 1 + } + if opts.xfer_dirs < 0 { + if opts.list_only != 0 { + opts.xfer_dirs = 1 + } else { + opts.xfer_dirs = 0 + } + } + + if opts.relative_paths < 0 { + if opts.files_from != "" { + opts.relative_paths = 1 + } else { + opts.relative_paths = 0 + } + } + + if opts.relative_paths == 0 { + opts.implied_dirs = 0 + } + + // NOTE: This simplification means that even if we ignore POPT_ARGFLAG_OR + // and store ints without regards for bit sets, we get the same result. + // Nevertheless, we support bit to be future-proof as new options are added. + if opts.missing_args == 3 { + // simplify if both options were specified + opts.missing_args = 2 + } + + if opts.backup_suffix == "" && opts.backup_dir == "" { + opts.backup_suffix = "~" + } + + if opts.backup_dir != "" { + opts.make_backups = 1 // --backup-dir implies --backup + } + + if opts.do_progress != 0 && opts.am_server == 0 { + if opts.info[INFO_NAME] == 0 { + opts.info[INFO_NAME] = 1 + } + opts.info[INFO_FLIST] = 2 + opts.info[INFO_PROGRESS] = 1 + } + + if opts.info[INFO_NAME] >= 1 && opts.stdout_format == "" { + opts.stdout_format = "%n%L" + } + + return nil +} diff --git a/internal/rsyncopts/rsyncopts_test.go b/internal/rsyncopts/rsyncopts_test.go new file mode 100644 index 0000000..10e3d8b --- /dev/null +++ b/internal/rsyncopts/rsyncopts_test.go @@ -0,0 +1,234 @@ +package rsyncopts + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync/internal/rsyncostest" +) + +// See testdata/_tridge_rsync_dump_table.patch for the corresponding +// C code (dump_long_options()) we use to compare the parsing result. +func dumpTable(buf *strings.Builder, pc *Context) { + for _, opt := range pc.table { + longName := opt.longName + if longName == "" { + longName = "(null)" + } + fmt.Fprintf(buf, "long=%s short=%s arg=", longName, opt.shortName) + switch opt.argInfo { + case POPT_ARG_STRING: + if opt.arg == nil { + fmt.Fprintf(buf, "\"(null)\"\n") + } else { + stringPtr := opt.arg.(*string) + if stringPtr == nil { + fmt.Fprintf(buf, "\"(null)\"\n") + } else { + if *stringPtr == "" { + fmt.Fprintf(buf, "\"(null)\"\n") + } else { + fmt.Fprintf(buf, "%q\n", *stringPtr) + } + } + } + case POPT_ARG_NONE, + POPT_ARG_INT, + POPT_ARG_VAL, + POPT_BIT_SET: + if opt.arg == nil { + fmt.Fprintf(buf, "\n") + } else { + intPtr := opt.arg.(*int) + if intPtr == nil { + fmt.Fprintf(buf, "\n") + } else { + fmt.Fprintf(buf, "%d\n", *intPtr) + } + } + default: + fmt.Fprintf(buf, "unknown argInfo: %v", opt.argInfo) + } + } +} + +func discardKnownDifferences(lines []string) []string { + filtered := make([]string, 0, len(lines)) + for _, line := range lines { + // TODO: document why ipv4/ipv6 have different values + ignore := strings.HasPrefix(line, "long=ipv4 ") || + strings.HasPrefix(line, "long=ipv6 ") || + // We implement protocol version 27 currently, + // tridge rsync implements newer versions. + strings.HasPrefix(line, "long=protocol ") || + // gokrazy-specific flags + strings.HasPrefix(line, "long=gokr.") + if !ignore { + filtered = append(filtered, line) + } + } + return filtered +} + +func TestParseArguments(t *testing.T) { + for _, tt := range []struct { + args []string + goldenTable string + }{ + { + // --recursive (-r) is ARG_VAL with val==2 + // --delete is ARG_NONE + args: []string{"-rtO", "--delete"}, + goldenTable: "rsync-rtO--delete.txt", + }, + { + args: []string{"--backup-dir=/tmp/backups"}, + goldenTable: "rsync-backup-dir.txt", + }, + { + // --temp-dir (-T) is ARG_STRING + args: []string{"-T", "/tmp"}, + goldenTable: "rsync-T-tmp.txt", + }, + { + args: []string{"-T=/tmp"}, + goldenTable: "rsync-T_tmp.txt", + }, + { + args: []string{"-T/tmp"}, + goldenTable: "rsync-Ttmp.txt", + }, + { + // --checksum-seed is ARG_INT + args: []string{"--checksum-seed", "2342"}, + goldenTable: "rsync-checksum-seed-2342.txt", + }, + { + // --delete-missing-args is BIT_SET + args: []string{"--delete-missing-args"}, + goldenTable: "rsync-delete-missing-args.txt", + }, + { + args: []string{"--ignore-missing-args"}, + goldenTable: "rsync-ignore-missing-args.txt", + }, + { + args: []string{"--ignore-missing-args", "--delete-missing-args"}, + goldenTable: "rsync-ignore-delete-missing-args.txt", + }, + { + args: []string{"--no-motd"}, + goldenTable: "rsync-no-motd.txt", + }, + { + // --verbose (-v) has val='v' + args: []string{"-vvv"}, + goldenTable: "rsync-vvv.txt", + }, + { + args: []string{"-a"}, + goldenTable: "rsync-a.txt", + }, + { + args: []string{"-P"}, + goldenTable: "rsync-P.txt", + }, + { + args: []string{"--debug=del2,acl"}, + goldenTable: "rsync-debug-del2-acl.txt", + }, + { + args: []string{"--info=name2"}, + goldenTable: "rsync-info-name2.txt", + }, + } { + t.Run(strings.Join(tt.args, " "), func(t *testing.T) { + want, err := os.ReadFile(filepath.Join("testdata", tt.goldenTable)) + if err != nil { + t.Fatal(err) + } + wantLines := discardKnownDifferences(strings.Split(strings.TrimSpace(string(want)), "\n")) + + osenv := rsyncostest.New(t) + pc := NewContext(NewOptions(osenv)) + if err := pc.ParseArguments(osenv, tt.args); err != nil { + t.Fatalf("ParseArguments: %v", err) + } + + var buf strings.Builder + dumpTable(&buf, pc) + gotLines := discardKnownDifferences(strings.Split(strings.TrimSpace(buf.String()), "\n")) + + if diff := cmp.Diff(wantLines, gotLines); diff != "" { + t.Errorf("unexpected option state: diff (-tridge +gokrazy):\n%s", diff) + } + }) + } +} + +func TestParseArgumentsError(t *testing.T) { + for _, tt := range []struct { + args []string + want int32 + wantMessage string + }{ + { + args: []string{"--delete=thoroughly"}, + want: POPT_ERROR_UNWANTEDARG, + }, + { + args: []string{"--does-not-exist"}, + want: POPT_ERROR_BADOPT, + wantMessage: "--does-not-exist: unknown option", + }, + { + args: []string{"-Q"}, + want: POPT_ERROR_BADOPT, + wantMessage: "-Q: unknown option", + }, + } { + t.Run(strings.Join(tt.args, " "), func(t *testing.T) { + osenv := rsyncostest.New(t) + pc := NewContext(NewOptions(osenv)) + err := pc.ParseArguments(osenv, tt.args) + if err == nil { + t.Fatalf("ParseArguments unexpectedly did not fail!") + } + got := err.(*PoptError).Errno + if got != tt.want { + t.Errorf("unexpected error: got %q, want %q", got, tt.want) + } + if tt.wantMessage != "" && !strings.Contains(err.Error(), tt.wantMessage) { + t.Errorf("unexpected error: got %q, want something containing %q", err.Error(), tt.wantMessage) + } + }) + } +} + +func TestParseArgumentsRemaining(t *testing.T) { + for _, tt := range []struct { + args []string + want []string + }{ + { + args: []string{"-aH", "-e", "./rsync.test", "localhost:/tmp/src/", "/tmp/dst"}, + want: []string{"localhost:/tmp/src/", "/tmp/dst"}, + }, + } { + t.Run(strings.Join(tt.args, " "), func(t *testing.T) { + osenv := rsyncostest.New(t) + pc := NewContext(NewOptions(osenv)) + if err := pc.ParseArguments(osenv, tt.args); err != nil { + t.Fatalf("ParseArguments: %v", err) + } + got := pc.RemainingArgs + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Errorf("RemainingArgs: unexpected diff (-want +got):\n%s", diff) + } + }) + } +} diff --git a/internal/rsyncopts/serveroptions.go b/internal/rsyncopts/serveroptions.go new file mode 100644 index 0000000..bd8cb45 --- /dev/null +++ b/internal/rsyncopts/serveroptions.go @@ -0,0 +1,221 @@ +package rsyncopts + +func (o *Options) CommandOptions(path string, paths ...string) []string { + return append(o.ServerOptions(), append([]string{".", path}, paths...)...) +} + +// rsync/options.c:server_options +func (o *Options) ServerOptions() []string { + var sargv []string + + // if (blocking_io == -1) + // blocking_io = 0; + + sargv = append(sargv, "--server") + + // if (daemon_over_rsh) { + // args[ac++] = "--daemon"; + // *argc = ac; + // /* if we're passing --daemon, we're done */ + // return; + // } + + if !o.Sender() { + sargv = append(sargv, "--sender") + } + + argstr := "-" + + // TODO: support verbosity levels, i.e. one or more -v + if o.Verbose() { + argstr += "v" + } + + // /* the -q option is intentionally left out */ + // if (make_backups) + // argstr[x++] = 'b'; + if o.UpdateOnly() { + argstr += "u" + } + if o.DryRun() { + argstr += "n" + } + if o.PreserveLinks() { + argstr += "l" + } + // if (copy_links) + // argstr[x++] = 'L'; + + // if (whole_file > 0) + // argstr[x++] = 'W'; + // /* We don't need to send --no-whole-file, because it's the + // * default for remote transfers, and in any case old versions + // * of rsync will not understand it. */ + + // if (preserve_hard_links) + // argstr[x++] = 'H'; + if o.PreserveUid() { + argstr += "o" + } + if o.PreserveGid() { + argstr += "g" + } + if o.PreserveDevices() { + argstr += "D" + } + if o.PreserveMTimes() { + argstr += "t" + } + if o.PreservePerms() { + argstr += "p" + } + if o.Recurse() { + argstr += "r" + } + if o.AlwaysChecksum() { + argstr += "c" + } + // if (cvs_exclude) + // argstr[x++] = 'C'; + if o.IgnoreTimes() { + argstr += "I" + } + // if (relative_paths) + // argstr[x++] = 'R'; + // if (one_file_system) + // argstr[x++] = 'x'; + // if (sparse_files) + // argstr[x++] = 'S'; + // if (do_compression) + // argstr[x++] = 'z'; + + // /* this is a complete hack - blame Rusty + + // this is a hack to make the list_only (remote file list) + // more useful */ + // if (list_only && !recurse) + // argstr[x++] = 'r'; + + // argstr[x] = 0; + + if argstr != "-" { + sargv = append(sargv, argstr) + } + + if o.Sender() && o.DoFsync() { + sargv = append(sargv, "--fsync") + } + + // if (block_size) { + // if (asprintf(&arg, "-B%u", block_size) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (max_delete && am_sender) { + // if (asprintf(&arg, "--max-delete=%d", max_delete) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (batch_prefix) { + // char *r_or_w = write_batch ? "write" : "read"; + // if (asprintf(&arg, "--%s-batch=%s", r_or_w, batch_prefix) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (io_timeout) { + // if (asprintf(&arg, "--timeout=%d", io_timeout) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (bwlimit) { + // if (asprintf(&arg, "--bwlimit=%d", bwlimit) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (backup_dir) { + // args[ac++] = "--backup-dir"; + // args[ac++] = backup_dir; + // } + + // /* Only send --suffix if it specifies a non-default value. */ + // if (strcmp(backup_suffix, backup_dir ? "" : BACKUP_SUFFIX) != 0) { + // /* We use the following syntax to avoid weirdness with '~'. */ + // if (asprintf(&arg, "--suffix=%s", backup_suffix) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (delete_excluded) + // args[ac++] = "--delete-excluded"; + // else if (delete_mode) + // args[ac++] = "--delete"; + + // if (size_only) + // args[ac++] = "--size-only"; + + // if (modify_window_set) { + // if (asprintf(&arg, "--modify-window=%d", modify_window) < 0) + // goto oom; + // args[ac++] = arg; + // } + + // if (keep_partial) + // args[ac++] = "--partial"; + + // if (force_delete) + // args[ac++] = "--force"; + + // if (delete_after) + // args[ac++] = "--delete-after"; + + // if (ignore_errors) + // args[ac++] = "--ignore-errors"; + + // if (copy_unsafe_links) + // args[ac++] = "--copy-unsafe-links"; + + // if (safe_symlinks) + // args[ac++] = "--safe-links"; + + // if (numeric_ids) + // args[ac++] = "--numeric-ids"; + + // if (only_existing && am_sender) + // args[ac++] = "--existing"; + + // if (opt_ignore_existing && am_sender) + // args[ac++] = "--ignore-existing"; + + // if (tmpdir) { + // args[ac++] = "--temp-dir"; + // args[ac++] = tmpdir; + // } + + // if (compare_dest && am_sender) { + // /* the server only needs this option if it is not the sender, + // * and it may be an older version that doesn't know this + // * option, so don't send it if client is the sender. + // */ + // args[ac++] = link_dest ? "--link-dest" : "--compare-dest"; + // args[ac++] = compare_dest; + // } + + // if (files_from && (!am_sender || remote_filesfrom_file)) { + // if (remote_filesfrom_file) { + // args[ac++] = "--files-from"; + // args[ac++] = remote_filesfrom_file; + // if (eol_nulls) + // args[ac++] = "--from0"; + // } else { + // args[ac++] = "--files-from=-"; + // args[ac++] = "--from0"; + // } + // } + + return sargv +} diff --git a/internal/rsyncopts/testdata/_tridge_rsync_dump_table.patch b/internal/rsyncopts/testdata/_tridge_rsync_dump_table.patch new file mode 100644 index 0000000..f41cf17 --- /dev/null +++ b/internal/rsyncopts/testdata/_tridge_rsync_dump_table.patch @@ -0,0 +1,68 @@ +diff --git i/main.c w/main.c +index 4f070acc..a4344402 100644 +--- i/main.c ++++ w/main.c +@@ -1781,6 +1781,8 @@ int main(int argc,char *argv[]) + option_error(); + exit_cleanup(RERR_SYNTAX); + } ++ dump_long_options(); ++ exit(23); + if (write_batch + && poptDupArgv(argc, (const char **)argv, &cooked_argc, (const char ***)&cooked_argv) != 0) + out_of_memory("main"); +diff --git i/options.c w/options.c +index 578507c6..32f60a95 100644 +--- i/options.c ++++ w/options.c +@@ -844,6 +844,50 @@ static struct poptOption long_options[] = { + {0,0,0,0, 0, 0, 0} + }; + ++void dump_long_options(void) ++{ ++ for (int i = 0; i < 251; i++) { ++ struct poptOption opt = long_options[i]; ++ printf("long=%s ", opt.longName); ++ if (opt.shortName == 0) { ++ printf("short= "); ++ } else { ++ printf("short=%c ", opt.shortName); ++ } ++ printf("arg="); ++ switch (opt.argInfo) { ++ case POPT_ARG_STRING: { ++ const char** ptr = opt.arg; ++ if (ptr == NULL) { ++ printf("\"(null)\"\n"); ++ } else { ++ if (*ptr == NULL || strlen(*ptr) == 0) { ++ printf("\"(null)\"\n"); ++ } else { ++ printf("\"%s\"\n", *ptr); ++ } ++ } ++ break; ++ } ++ ++ case POPT_ARG_NONE: ++ case POPT_ARG_INT: ++ case POPT_BIT_SET: ++ case POPT_ARG_VAL: { ++ const int* ptr = opt.arg; ++ if (ptr == NULL) { ++ printf("\n"); ++ } else { ++ printf("%d\n", *ptr); ++ } ++ break; ++ } ++ default: ++ printf("unknown argInfo: %d", opt.argInfo); ++ } ++ } ++} ++ + static struct poptOption long_daemon_options[] = { + /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */ + {"address", 0, POPT_ARG_STRING, &bind_address, 0, 0, 0 }, diff --git a/internal/rsyncopts/testdata/rsync-P.txt b/internal/rsyncopts/testdata/rsync-P.txt new file mode 100644 index 0000000..00c2383 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-P.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=1 +long=no-progress short= arg=1 +long=partial short= arg=1 +long=no-partial short= arg=1 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="%n%L" +long=log-format short= arg="%n%L" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-T-tmp.txt b/internal/rsyncopts/testdata/rsync-T-tmp.txt new file mode 100644 index 0000000..54a15b9 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-T-tmp.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="/tmp" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-T_tmp.txt b/internal/rsyncopts/testdata/rsync-T_tmp.txt new file mode 100644 index 0000000..54a15b9 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-T_tmp.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="/tmp" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-Ttmp.txt b/internal/rsyncopts/testdata/rsync-Ttmp.txt new file mode 100644 index 0000000..54a15b9 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-Ttmp.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="/tmp" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-a.txt b/internal/rsyncopts/testdata/rsync-a.txt new file mode 100644 index 0000000..ea6422a --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-a.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=1 +long=no-recursive short= arg=1 +long=no-r short= arg=1 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=1 +long=no-dirs short= arg=1 +long=no-d short= arg=1 +long=old-dirs short= arg=1 +long=old-d short= arg=1 +long=perms short=p arg=1 +long=no-perms short= arg=1 +long=no-p short= arg=1 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=1 +long=no-times short= arg=1 +long=no-t short= arg=1 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=1 +long=no-owner short= arg=1 +long=no-o short= arg=1 +long=group short=g arg=1 +long=no-group short= arg=1 +long=no-g short= arg=1 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=1 +long=no-devices short= arg=1 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=1 +long=no-specials short= arg=1 +long=links short=l arg=1 +long=no-links short= arg=1 +long=no-l short= arg=1 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-backup-dir.txt b/internal/rsyncopts/testdata/rsync-backup-dir.txt new file mode 100644 index 0000000..b31b330 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-backup-dir.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=1 +long=no-backup short= arg=1 +long=backup-dir short= arg="/tmp/backups" +long=suffix short= arg="(null)" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-checksum-seed-2342.txt b/internal/rsyncopts/testdata/rsync-checksum-seed-2342.txt new file mode 100644 index 0000000..1546b56 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-checksum-seed-2342.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=2342 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-debug-del2-acl.txt b/internal/rsyncopts/testdata/rsync-debug-del2-acl.txt new file mode 100644 index 0000000..22d5179 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-debug-del2-acl.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-delete-missing-args.txt b/internal/rsyncopts/testdata/rsync-delete-missing-args.txt new file mode 100644 index 0000000..84fdb48 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-delete-missing-args.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=2 +long=ignore-missing-args short= arg=2 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-ignore-delete-missing-args.txt b/internal/rsyncopts/testdata/rsync-ignore-delete-missing-args.txt new file mode 100644 index 0000000..84fdb48 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-ignore-delete-missing-args.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=2 +long=ignore-missing-args short= arg=2 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-ignore-missing-args.txt b/internal/rsyncopts/testdata/rsync-ignore-missing-args.txt new file mode 100644 index 0000000..1ef2fcb --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-ignore-missing-args.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=1 +long=ignore-missing-args short= arg=1 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-info-name2.txt b/internal/rsyncopts/testdata/rsync-info-name2.txt new file mode 100644 index 0000000..e95ed0b --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-info-name2.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="%n%L" +long=log-format short= arg="%n%L" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-no-motd.txt b/internal/rsyncopts/testdata/rsync-no-motd.txt new file mode 100644 index 0000000..f4c3258 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-no-motd.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=0 +long=no-motd short= arg=0 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-rtO--delete.txt b/internal/rsyncopts/testdata/rsync-rtO--delete.txt new file mode 100644 index 0000000..bb71949 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-rtO--delete.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=0 +long=no-v short= arg=0 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=2 +long=no-recursive short= arg=2 +long=no-r short= arg=2 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=1 +long=no-dirs short= arg=1 +long=no-d short= arg=1 +long=old-dirs short= arg=1 +long=old-d short= arg=1 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=1 +long=no-times short= arg=1 +long=no-t short= arg=1 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=1 +long=no-omit-dir-times short= arg=1 +long=no-O short= arg=1 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=1 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="(null)" +long=log-format short= arg="(null)" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncopts/testdata/rsync-vvv.txt b/internal/rsyncopts/testdata/rsync-vvv.txt new file mode 100644 index 0000000..d2df7d2 --- /dev/null +++ b/internal/rsyncopts/testdata/rsync-vvv.txt @@ -0,0 +1,251 @@ +long=help short= arg= +long=version short=V arg= +long=verbose short=v arg= +long=no-verbose short= arg=3 +long=no-v short= arg=3 +long=info short= arg="(null)" +long=debug short= arg="(null)" +long=stderr short= arg="(null)" +long=msgs2stderr short= arg=2 +long=no-msgs2stderr short= arg=2 +long=quiet short=q arg= +long=motd short= arg=1 +long=no-motd short= arg=1 +long=stats short= arg=0 +long=human-readable short=h arg= +long=no-human-readable short= arg=1 +long=no-h short= arg=1 +long=dry-run short=n arg=0 +long=archive short=a arg= +long=recursive short=r arg=0 +long=no-recursive short= arg=0 +long=no-r short= arg=0 +long=inc-recursive short= arg=1 +long=no-inc-recursive short= arg=1 +long=i-r short= arg=1 +long=no-i-r short= arg=1 +long=dirs short=d arg=0 +long=no-dirs short= arg=0 +long=no-d short= arg=0 +long=old-dirs short= arg=0 +long=old-d short= arg=0 +long=perms short=p arg=0 +long=no-perms short= arg=0 +long=no-p short= arg=0 +long=executability short=E arg=0 +long=acls short=A arg= +long=no-acls short= arg=0 +long=no-A short= arg=0 +long=xattrs short=X arg= +long=no-xattrs short= arg=0 +long=no-X short= arg=0 +long=times short=t arg=0 +long=no-times short= arg=0 +long=no-t short= arg=0 +long=atimes short=U arg= +long=no-atimes short= arg=0 +long=no-U short= arg=0 +long=open-noatime short= arg=0 +long=no-open-noatime short= arg=0 +long=crtimes short=N arg=0 +long=no-crtimes short= arg=0 +long=no-N short= arg=0 +long=omit-dir-times short=O arg=0 +long=no-omit-dir-times short= arg=0 +long=no-O short= arg=0 +long=omit-link-times short=J arg=0 +long=no-omit-link-times short= arg=0 +long=no-J short= arg=0 +long=modify-window short=@ arg=0 +long=super short= arg=0 +long=no-super short= arg=0 +long=fake-super short= arg=0 +long=owner short=o arg=0 +long=no-owner short= arg=0 +long=no-o short= arg=0 +long=group short=g arg=0 +long=no-group short= arg=0 +long=no-g short= arg=0 +long=(null) short=D arg= +long=no-D short= arg= +long=devices short= arg=0 +long=no-devices short= arg=0 +long=copy-devices short= arg=0 +long=write-devices short= arg=0 +long=no-write-devices short= arg=0 +long=specials short= arg=0 +long=no-specials short= arg=0 +long=links short=l arg=0 +long=no-links short= arg=0 +long=no-l short= arg=0 +long=copy-links short=L arg=0 +long=copy-unsafe-links short= arg=0 +long=safe-links short= arg=0 +long=munge-links short= arg=0 +long=no-munge-links short= arg=0 +long=copy-dirlinks short=k arg=0 +long=keep-dirlinks short=K arg=0 +long=hard-links short=H arg= +long=no-hard-links short= arg=0 +long=no-H short= arg=0 +long=relative short=R arg=0 +long=no-relative short= arg=0 +long=no-R short= arg=0 +long=implied-dirs short= arg=0 +long=no-implied-dirs short= arg=0 +long=i-d short= arg=0 +long=no-i-d short= arg=0 +long=chmod short= arg="(null)" +long=ignore-times short=I arg=0 +long=size-only short= arg=0 +long=one-file-system short=x arg= +long=no-one-file-system short= arg=0 +long=no-x short= arg=0 +long=update short=u arg=0 +long=existing short= arg=0 +long=ignore-non-existing short= arg=0 +long=ignore-existing short= arg=0 +long=max-size short= arg="(null)" +long=min-size short= arg="(null)" +long=max-alloc short= arg="(null)" +long=sparse short=S arg=0 +long=no-sparse short= arg=0 +long=no-S short= arg=0 +long=preallocate short= arg=0 +long=inplace short= arg=0 +long=no-inplace short= arg=0 +long=append short= arg= +long=append-verify short= arg=0 +long=no-append short= arg=0 +long=del short= arg=0 +long=delete short= arg=0 +long=delete-before short= arg=0 +long=delete-during short= arg=0 +long=delete-delay short= arg=0 +long=delete-after short= arg=0 +long=delete-excluded short= arg=0 +long=delete-missing-args short= arg=0 +long=ignore-missing-args short= arg=0 +long=remove-sent-files short= arg=0 +long=remove-source-files short= arg=0 +long=force short= arg=0 +long=no-force short= arg=0 +long=ignore-errors short= arg=0 +long=no-ignore-errors short= arg=0 +long=max-delete short= arg=-2147483648 +long=(null) short=F arg= +long=filter short=f arg="(null)" +long=exclude short= arg="(null)" +long=include short= arg="(null)" +long=exclude-from short= arg="(null)" +long=include-from short= arg="(null)" +long=cvs-exclude short=C arg=0 +long=whole-file short=W arg=-1 +long=no-whole-file short= arg=-1 +long=no-W short= arg=-1 +long=checksum short=c arg=0 +long=no-checksum short= arg=0 +long=no-c short= arg=0 +long=checksum-choice short= arg="(null)" +long=cc short= arg="(null)" +long=block-size short=B arg="(null)" +long=compare-dest short= arg="(null)" +long=copy-dest short= arg="(null)" +long=link-dest short= arg="(null)" +long=fuzzy short=y arg= +long=no-fuzzy short= arg=0 +long=no-y short= arg=0 +long=compress short=z arg= +long=old-compress short= arg= +long=new-compress short= arg= +long=no-compress short= arg= +long=no-z short= arg= +long=compress-choice short= arg="(null)" +long=zc short= arg="(null)" +long=skip-compress short= arg="(null)" +long=compress-level short= arg=-2147483648 +long=zl short= arg=-2147483648 +long=(null) short=P arg= +long=progress short= arg=0 +long=no-progress short= arg=0 +long=partial short= arg=0 +long=no-partial short= arg=0 +long=partial-dir short= arg="(null)" +long=delay-updates short= arg=0 +long=no-delay-updates short= arg=0 +long=prune-empty-dirs short=m arg=0 +long=no-prune-empty-dirs short= arg=0 +long=no-m short= arg=0 +long=log-file short= arg="(null)" +long=log-file-format short= arg="(null)" +long=out-format short= arg="%n%L" +long=log-format short= arg="%n%L" +long=itemize-changes short=i arg= +long=no-itemize-changes short= arg=0 +long=no-i short= arg=0 +long=bwlimit short= arg="(null)" +long=no-bwlimit short= arg=0 +long=backup short=b arg=0 +long=no-backup short= arg=0 +long=backup-dir short= arg="(null)" +long=suffix short= arg="~" +long=list-only short= arg=0 +long=read-batch short= arg="(null)" +long=write-batch short= arg="(null)" +long=only-write-batch short= arg="(null)" +long=files-from short= arg="(null)" +long=from0 short=0 arg=0 +long=no-from0 short= arg=0 +long=old-args short= arg= +long=no-old-args short= arg=0 +long=secluded-args short=s arg=0 +long=no-secluded-args short= arg=0 +long=protect-args short= arg=0 +long=no-protect-args short= arg=0 +long=no-s short= arg=0 +long=trust-sender short= arg=0 +long=numeric-ids short= arg=0 +long=no-numeric-ids short= arg=0 +long=usermap short= arg="(null)" +long=groupmap short= arg="(null)" +long=chown short= arg="(null)" +long=timeout short= arg=0 +long=no-timeout short= arg=0 +long=contimeout short= arg=0 +long=no-contimeout short= arg=0 +long=fsync short= arg=0 +long=stop-after short= arg="(null)" +long=time-limit short= arg="(null)" +long=stop-at short= arg="(null)" +long=rsh short=e arg="(null)" +long=rsync-path short= arg="rsync" +long=temp-dir short=T arg="(null)" +long=iconv short= arg="(null)" +long=no-iconv short= arg= +long=ipv4 short=4 arg=0 +long=ipv6 short=6 arg=0 +long=8-bit-output short=8 arg=0 +long=no-8-bit-output short= arg=0 +long=no-8 short= arg=0 +long=mkpath short= arg=0 +long=no-mkpath short= arg=0 +long=qsort short= arg=0 +long=copy-as short= arg="(null)" +long=address short= arg="(null)" +long=port short= arg=0 +long=sockopts short= arg="(null)" +long=password-file short= arg="(null)" +long=early-input short= arg="(null)" +long=blocking-io short= arg=-1 +long=no-blocking-io short= arg=-1 +long=outbuf short= arg="(null)" +long=remote-option short=M arg="(null)" +long=protocol short= arg=32 +long=checksum-seed short= arg=0 +long=server short= arg= +long=sender short= arg= +long=config short= arg="(null)" +long=daemon short= arg= +long=dparam short= arg="(null)" +long=detach short= arg= +long=no-detach short= arg= diff --git a/internal/rsyncos/rsyncos.go b/internal/rsyncos/rsyncos.go new file mode 100644 index 0000000..daabc22 --- /dev/null +++ b/internal/rsyncos/rsyncos.go @@ -0,0 +1,39 @@ +package rsyncos + +import ( + "context" + "io" + "net" + + "github.com/runpod/rsync/internal/log" +) + +type Env struct { + Stdin io.ReadCloser + Stdout io.WriteCloser + Stderr io.WriteCloser + + DontRestrict bool + + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + logger log.Logger +} + +func (s *Env) initLogger() { + if s.logger == nil { + s.logger = log.New(s.Stderr) + } +} + +func (s *Env) Logger() log.Logger { + s.initLogger() + return s.logger +} + +func (s *Env) Logf(format string, v ...any) { + s.initLogger() + s.logger.Printf(format, v...) +} + +func (s *Env) Restrict() bool { return !s.DontRestrict } diff --git a/internal/rsyncostest/rsyncostest.go b/internal/rsyncostest/rsyncostest.go new file mode 100644 index 0000000..7a2cd97 --- /dev/null +++ b/internal/rsyncostest/rsyncostest.go @@ -0,0 +1,15 @@ +package rsyncostest + +import ( + "testing" + + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/testlogger" +) + +func New(t *testing.T) *rsyncos.Env { + return &rsyncos.Env{ + // Logs go to stderr, so wire that up to a testlogger. + Stderr: testlogger.New(t), + } +} diff --git a/internal/rsyncstats/rsyncstats.go b/internal/rsyncstats/rsyncstats.go new file mode 100644 index 0000000..18920fa --- /dev/null +++ b/internal/rsyncstats/rsyncstats.go @@ -0,0 +1,7 @@ +package rsyncstats + +type TransferStats struct { + Read int64 // total bytes read (from network connection) + Written int64 // total bytes written (to network connection) + Size int64 // total size of files +} diff --git a/internal/rsynctest/restrict.go b/internal/rsynctest/restrict.go new file mode 100644 index 0000000..fbb4251 --- /dev/null +++ b/internal/rsynctest/restrict.go @@ -0,0 +1,24 @@ +package rsynctest + +import ( + "os" + + "github.com/landlock-lsm/go-landlock/landlock" + "github.com/runpod/rsync/internal/restrict" +) + +func init() { + restrict.ExtraHook = func() []landlock.Rule { + return []landlock.Rule{ + // contains /usr/bin/rsync (and library deps) + landlock.RODirs("/usr"), + landlock.RODirs("/nix").IgnoreIfMissing(), + + // for t.TempDir() + landlock.RWDirs(os.TempDir()).WithRefer(), + + // used in some of our test code + landlock.RWFiles("/dev/null"), + } + } +} diff --git a/internal/rsynctest/rsynctest.go b/internal/rsynctest/rsynctest.go new file mode 100644 index 0000000..bb2f5d3 --- /dev/null +++ b/internal/rsynctest/rsynctest.go @@ -0,0 +1,465 @@ +package rsynctest + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/anonssh" + "github.com/runpod/rsync/internal/maincmd" + "github.com/runpod/rsync/internal/rsyncdconfig" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncostest" + "github.com/runpod/rsync/internal/rsyncstats" + "github.com/runpod/rsync/internal/testlogger" + "github.com/runpod/rsync/rsyncclient" + "github.com/runpod/rsync/rsynccmd" + "github.com/runpod/rsync/rsyncd" +) + +// GosPublicRelease is the time when Go was publicly released: +// https://opensource.googleblog.com/2009/11/hey-ho-lets-go.html +// +// It’s as good a time stamp as any, e.g. for setting an mtime. +var GosPublicRelease = func() time.Time { + t, err := time.Parse(time.RFC3339, "2009-11-10T23:00:00Z") + if err != nil { + panic(err) + } + return t +}() + +type TestServer struct { + // config + module rsyncd.Module + listener net.Listener + listeners []rsyncdconfig.Listener + dontRestrict bool + + // state + srv *rsyncd.Server + + // Port is the port on which the test server is listening on. Useful to pass + // to rsync’s --port option. + Port string +} + +// InteropModule is a convenience function to define an rsync module named +// “interop” with the specified path. +func InteropModule(path string) []rsyncd.Module { + return []rsyncd.Module{ + { + Name: "interop", + Path: path, + }, + } +} + +// WritableInteropModule is a wrapper around InteropModule that marks the module +// as writable (not read-only). +func WritableInteropModule(path string) []rsyncd.Module { + mods := InteropModule(path) + mods[0].Writable = true + return mods +} + +type Option func(ts *TestServer) + +func Listeners(lns []rsyncdconfig.Listener) Option { + return func(ts *TestServer) { + ts.listeners = lns + } +} + +func Listener(ln net.Listener) Option { + return func(ts *TestServer) { + ts.listener = ln + } +} + +func DontRestrict() Option { + return func(ts *TestServer) { + ts.dontRestrict = true + } +} + +func New(t *testing.T, modules []rsyncd.Module, opts ...Option) *TestServer { + ctx := t.Context() + + ts := &TestServer{} + for _, opt := range opts { + opt(ts) + } + if len(ts.listeners) == 0 { + ts.listeners = []rsyncdconfig.Listener{ + {Rsyncd: "localhost:0"}, + } + } + srv, err := rsyncd.NewServer(modules, rsyncd.WithStderr(testlogger.New(t)), rsyncd.DontRestrict()) + if err != nil { + t.Fatal(err) + } + + if ts.listener == nil { + ln, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + ts.listener = ln + } + + t.Logf("listening on %s", ts.listener.Addr()) + _, port, err := net.SplitHostPort(ts.listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + ts.Port = port + + osenv := rsyncostest.New(t) + if ts.listeners[0].AuthorizedSSH.Address != "" { + sshListener, err := anonssh.ListenerFromConfig(osenv, ts.listeners[0]) + if err != nil { + t.Fatal(err) + } + cfg := &rsyncdconfig.Config{ + Modules: modules, + } + go func() { + err := anonssh.Serve(ctx, osenv, ts.listener, sshListener, cfg, func(args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { + osenv := &rsyncos.Env{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } + _, err := maincmd.Main(context.Background(), osenv, args, cfg) + return err + }) + + if errors.Is(err, net.ErrClosed) { + return + } + + if err != nil { + t.Error(err) + } + }() + } else if ts.listeners[0].AnonSSH != "" { + sshListener, err := anonssh.ListenerFromConfig(osenv, ts.listeners[0]) + if err != nil { + t.Fatal(err) + } + cfg := &rsyncdconfig.Config{ + Modules: modules, + } + go func() { + err := anonssh.Serve(ctx, osenv, ts.listener, sshListener, cfg, func(args []string, stdin io.ReadCloser, stdout io.WriteCloser, stderr io.WriteCloser) error { + osenv := &rsyncos.Env{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + } + _, err := maincmd.Main(context.Background(), osenv, args, cfg) + return err + }) + + if errors.Is(err, net.ErrClosed) { + return + } + + if err != nil { + t.Error(err) + } + }() + } else { + go srv.Serve(context.Background(), ts.listener) + } + + return ts +} + +func Run(tb testing.TB, args ...string) *rsyncstats.TransferStats { + cmd := rsynccmd.Command(args[0], args[1:]...) + cmd.Stdout = testlogger.New(tb) + cmd.Stderr = testlogger.New(tb) + result, err := cmd.Run(tb.Context()) + if err != nil { + tb.Fatal(err) + } + return result.Stats +} + +type nopCloser struct{} + +func (*nopCloser) Close() error { return nil } + +func nopClose(w io.Writer) io.WriteCloser { + return struct { + io.Writer + io.Closer + }{ + Writer: w, + Closer: &nopCloser{}, + } +} + +func Output(tb testing.TB, args ...string) (stdout []byte, stderr []byte) { + tb.Helper() + var stdoutb, stderrb bytes.Buffer + cmd := rsynccmd.Command(args[0], args[1:]...) + cmd.Stdout = nopClose(&stdoutb) + cmd.Stderr = nopClose(&stderrb) + _, err := cmd.Run(context.Background()) + if err != nil { + tb.Fatal(err) + } + return stdoutb.Bytes(), stderrb.Bytes() +} + +func CombinedOutput(args ...string) ([]byte, error) { + var buf bytes.Buffer + cmd := rsynccmd.Command(args[0], args[1:]...) + cmd.Stdout = nopClose(&buf) + cmd.Stderr = nopClose(&buf) + _, err := cmd.Run(context.Background()) + return buf.Bytes(), err +} + +func NewInMemory(t *testing.T, module rsyncd.Module, opts ...Option) *TestServer { + ts := &TestServer{ + module: module, + } + for _, opt := range opts { + opt(ts) + } + + stderr := testlogger.New(t) + rsyncdOpts := []rsyncd.Option{ + rsyncd.WithStderr(stderr), + } + if ts.dontRestrict { + rsyncdOpts = append(rsyncdOpts, rsyncd.DontRestrict()) + } + srv, err := rsyncd.NewServer([]rsyncd.Module{module}, rsyncdOpts...) + if err != nil { + t.Fatal(err) + } + ts.srv = srv + return ts +} + +func (ts *TestServer) pipe(t *testing.T, args []string) (*sync.WaitGroup, io.ReadWriteCloser) { + // stdin from the view of the rsync server + stdinrd, stdinwr := io.Pipe() + stdoutrd, stdoutwr := io.Pipe() + conn := rsyncd.NewConnection(stdinrd, stdoutwr, "") + osenv := rsyncostest.New(t) + pc := rsyncopts.NewContext(rsyncopts.NewOptionsWithGokrazyDefaults(osenv)) + if err := pc.ParseArguments(osenv, args); err != nil { + t.Fatalf("parsing server args: %v", err) + } + t.Logf("pc.RemainingArgs=%q", pc.RemainingArgs) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := ts.srv.InternalHandleConn(t.Context(), conn, &ts.module, pc) + if err != nil { + t.Error(err) + } + }() + + rw := &rsync.BothCloser{ + ReadCloser: stdoutrd, + WriteCloser: stdinwr, + } + return &wg, rw +} + +func (ts *TestServer) RunClient(t *testing.T, args []string, src string, remaining []string) *rsyncstats.TransferStats { + stderr := testlogger.New(t) + cl, err := rsyncclient.New(args, rsyncclient.WithStderr(stderr), rsyncclient.DontRestrict()) + if err != nil { + t.Fatal(err) + } + wg, rw := ts.pipe(t, cl.ServerCommandOptions(src)) + res, err := cl.Run(t.Context(), rw, remaining) + if err != nil { + t.Fatal(err) + } + // Ensure an error would be displayed, if any. + wg.Wait() + return res.Stats +} + +func CommandMain(m *testing.M) error { + osenv := &rsyncos.Env{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + if len(os.Args) > 1 && os.Args[1] == "localhost" { + // Strip first 2 args (./rsync.test localhost) from command line: + // rsync(1) is calling this process as a remote shell. + os.Args = os.Args[2:] + if _, err := maincmd.Main(context.Background(), osenv, os.Args, nil); err != nil { + return err + } + } else if len(os.Args) > 1 && os.Args[1] == "--server" { + // gokr-rsync is calling this process as a local daemon. + if _, err := maincmd.Main(context.Background(), osenv, os.Args, nil); err != nil { + return err + } + } else { + os.Exit(m.Run()) + } + return nil +} + +func ConstructLargeDataFile(headPattern, bodyPattern, endPattern []byte) []byte { + // create large data file in source directory to be copied + head := bytes.Repeat(headPattern, 1*1024) + body := bytes.Repeat(bodyPattern, 1*1024) + end := bytes.Repeat(endPattern, 1*1024) + return append(append(head, body...), end...) +} + +func WriteLargeDataFile(t *testing.T, source string, headPattern, bodyPattern, endPattern []byte) { + // create large data file in source directory to be copied + content := ConstructLargeDataFile(headPattern, bodyPattern, endPattern) + large := filepath.Join(source, "large-data-file") + if err := os.MkdirAll(filepath.Dir(large), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(large, content, 0644); err != nil { + t.Fatal(err) + } +} + +func DataFileMatches(fn string, headPattern, bodyPattern, endPattern []byte) error { + want := ConstructLargeDataFile(headPattern, bodyPattern, endPattern) + got, err := os.ReadFile(fn) + if err != nil { + return err + } + // fast path: using bytes.Equal for an equality check uses much less + // resources compared to cmp.Diff. + if bytes.Equal(want, got) { + return nil + } + if diff := cmp.Diff(want, got); diff != "" { + return fmt.Errorf("unexpected file contents: diff (-want +got):\n%s", diff) + } + + return nil +} + +type discovered struct { + rsync string // path to any rsync, typically "rsync" + tridgeRsync string // path to tridge rsync if discovered +} + +func (d discovered) anyRsync() string { + if d.tridgeRsync != "" { + return d.tridgeRsync + } + return d.rsync +} + +var discoverOnce = sync.OnceValue(func() discovered { + // For tests that need tridge rsync, explicitly check + // a few well-known locations. + locations := []string{ + // If rsync is installed from homebrew, + // that will typically be the latest / preferred version. + "/opt/homebrew/bin/rsync", + } + if runtime.GOOS == "darwin" { + // macOS 15 replaced rsync with a wrapper that + // dispatches to openrsync by default, unless + // it finds an option that it knows needs tridge rsync. + // They will probably stop shipping tridge rsync + // eventually, but for now check this location: + locations = append(locations, "/usr/libexec/rsync/rsync.samba") + } + for _, loc := range locations { + if _, err := os.Stat(loc); err == nil { + return discovered{tridgeRsync: loc} + } + } + version, err := exec.Command("rsync", "--version").Output() + if err != nil { + return discovered{} + } + if strings.Contains(string(version), "openrsync:") { + return discovered{rsync: "rsync"} + } + return discovered{tridgeRsync: "rsync"} +}) + +func TridgeOrGTFO(t *testing.T, reason string) string { + discovered := discoverOnce() + if discovered.anyRsync() == "" { + // Gotta set some boundaries. + // We need *some* rsync. + t.Fatalf("no rsync installed") + } + if discovered.tridgeRsync == "" { + // we did not find tridge rsync and the default rsync is openrsync + t.Skipf("tridge rsync not found, cannot run this test: %v", reason) + } + return discovered.tridgeRsync +} + +func AnyRsync(t *testing.T) string { + any := discoverOnce().anyRsync() + if any == "" { + // Gotta set some boundaries. + // We need *some* rsync. + t.Fatalf("no rsync installed") + } + return any +} + +var rsyncVersionRe = regexp.MustCompile(`rsync\s*version ([v0-9.]+)`) +var rsyncVersionOnce = sync.OnceValue(func() string { + any := discoverOnce().anyRsync() + version := exec.Command(any, "--version") + version.Stderr = os.Stderr + b, err := version.Output() + if err != nil { + return fmt.Sprintf("BUG: %s --version: %v", any, err) + } + matches := rsyncVersionRe.FindStringSubmatch(string(b)) + if len(matches) == 0 { + return fmt.Sprintf("BUG: rsync version number not found in output %q", string(b)) + } + // rsync 2.6.9 does not print a v prefix, + // but rsync v3.2.3 does print a v prefix. + return strings.TrimPrefix(matches[1], "v") +}) + +func RsyncVersion(t *testing.T) string { + any := discoverOnce().anyRsync() + if any == "" { + // Gotta set some boundaries. + // We need *some* rsync. + t.Fatalf("no rsync installed") + } + return rsyncVersionOnce() +} diff --git a/internal/rsynctest/rsynctest_unix.go b/internal/rsynctest/rsynctest_unix.go new file mode 100644 index 0000000..e504ad0 --- /dev/null +++ b/internal/rsynctest/rsynctest_unix.go @@ -0,0 +1,140 @@ +//go:build !windows + +package rsynctest + +import ( + "net" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/google/renameio/v2" + "golang.org/x/sys/unix" +) + +func CreateDummyDeviceFiles(t *testing.T, dir string) { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + + char := filepath.Join(dir, "char") + // major 1, minor 5, like /dev/zero + if err := unix.Mknod(char, 0600|syscall.S_IFCHR, int(unix.Mkdev(1, 5))); err != nil { + t.Fatal(err) + } + + block := filepath.Join(dir, "block") + // major 242, minor 9, like /dev/nvme0 + if err := unix.Mknod(block, 0600|syscall.S_IFBLK, int(unix.Mkdev(242, 9))); err != nil { + t.Fatal(err) + } + + fifo := filepath.Join(dir, "fifo") + if err := unix.Mkfifo(fifo, 0600); err != nil { + t.Fatal(err) + } + + sock := filepath.Join(dir, "sock") + ln, err := net.Listen("unix", sock) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) +} + +func VerifyDummyDeviceFiles(t *testing.T, source, dest string) { + { + sourcest, err := os.Stat(filepath.Join(source, "char")) + if err != nil { + t.Fatal(err) + } + destst, err := os.Stat(filepath.Join(dest, "char")) + if err != nil { + t.Fatal(err) + } + if destst.Mode().Type()&os.ModeCharDevice == 0 { + t.Fatalf("unexpected type: got %v, want character device", destst.Mode()) + } + destsys, ok := destst.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("stat does not contain rdev") + } + sourcesys, ok := sourcest.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("stat does not contain rdev") + } + if got, want := destsys.Rdev, sourcesys.Rdev; got != want { + t.Fatalf("unexpected rdev: got %v, want %v", got, want) + } + } + + { + sourcest, err := os.Stat(filepath.Join(source, "block")) + if err != nil { + t.Fatal(err) + } + destst, err := os.Stat(filepath.Join(dest, "block")) + if err != nil { + t.Fatal(err) + } + if destst.Mode().Type()&os.ModeDevice == 0 || + destst.Mode().Type()&os.ModeCharDevice != 0 { + t.Fatalf("unexpected type: got %v, want block device", destst.Mode()) + } + destsys, ok := destst.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("stat does not contain rdev") + } + sourcesys, ok := sourcest.Sys().(*syscall.Stat_t) + if !ok { + t.Fatal("stat does not contain rdev") + } + if got, want := destsys.Rdev, sourcesys.Rdev; got != want { + t.Fatalf("unexpected rdev: got %v, want %v", got, want) + } + } + + { + st, err := os.Stat(filepath.Join(dest, "fifo")) + if err != nil { + t.Fatal(err) + } + if st.Mode().Type()&os.ModeNamedPipe == 0 { + t.Fatalf("unexpected type: got %v, want fifo", st.Mode()) + } + } + + { + st, err := os.Stat(filepath.Join(dest, "sock")) + if err != nil { + t.Fatal(err) + } + if st.Mode().Type()&os.ModeSocket == 0 { + t.Fatalf("unexpected type: got %v, want socket", st.Mode()) + } + } +} + +// StatUidGid returns the owner uid and gid backing fi. +// +// This check is encapsulated in StatUidGid so that +// on Windows, we can fail loudly (the check should be skipped). +func StatUidGid(t *testing.T, fi os.FileInfo) (uid, gid int) { + stt, ok := fi.Sys().(*syscall.Stat_t) + if !ok { + t.Fatalf("FileInfo.Sys() is %T, want *syscall.Stat_t", fi.Sys()) + } + return int(stt.Uid), int(stt.Gid) +} + +// ReplaceSymlink atomically replaces newname with a symlink to oldname. +// +// On Unix, this function works atomically, +// on Windows it falls back to non-atomic behavior. +func ReplaceSymlink(t *testing.T, oldname, newname string) { + t.Helper() + if err := renameio.Symlink(oldname, newname); err != nil { + t.Fatal(err) + } +} diff --git a/internal/rsynctest/rsynctest_windows.go b/internal/rsynctest/rsynctest_windows.go new file mode 100644 index 0000000..3f71004 --- /dev/null +++ b/internal/rsynctest/rsynctest_windows.go @@ -0,0 +1,39 @@ +//go:build windows + +package rsynctest + +import ( + "os" + "testing" +) + +func CreateDummyDeviceFiles(t *testing.T, dir string) { + t.Skip("Windows does not support special files (devices/FIFOs/sockets)") +} + +func VerifyDummyDeviceFiles(t *testing.T, source, dest string) { + t.Skip("Windows does not support special files (devices/FIFOs/sockets)") +} + +// StatUidGid is not meaningful on Windows (no Unix file ownership). +// +// Callers are supposed to check os.Getuid()==-1 and skip. +// Hence, this function must compile but is never reached. +func StatUidGid(t *testing.T, fi os.FileInfo) (uid, gid int) { + t.Fatal("BUG: StatUidGid not skipped on Windows") + return 0, 0 +} + +// ReplaceSymlink atomically replaces newname with a symlink to oldname. +// +// On Unix, this function works atomically, +// on Windows it falls back to non-atomic behavior. +func ReplaceSymlink(t *testing.T, oldname, newname string) { + t.Helper() + if err := os.Remove(newname); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + if err := os.Symlink(oldname, newname); err != nil { + t.Skipf("cannot create symlinks on this Windows host: %v", err) + } +} diff --git a/internal/rsyncwire/wire.go b/internal/rsyncwire/wire.go new file mode 100644 index 0000000..6792b49 --- /dev/null +++ b/internal/rsyncwire/wire.go @@ -0,0 +1,238 @@ +package rsyncwire + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/runpod/rsync/internal/rsyncos" +) + +const ( + MsgData uint8 = 0 + MsgInfo uint8 = 2 + MsgError uint8 = 1 +) + +const mplexBase = 7 + +type MultiplexWriter struct { + Writer io.WriteCloser +} + +func (w *MultiplexWriter) Write(p []byte) (n int, err error) { + return w.WriteMsg(MsgData, p) +} + +func (w *MultiplexWriter) WriteMsg(tag uint8, p []byte) (n int, err error) { + header := uint32(mplexBase+tag)<<24 | uint32(len(p)) + // log.Printf("len %d (hex %x)", len(p), uint32(len(p))) + // log.Printf("header=%v (%x)", header, header) + if err := binary.Write(w.Writer, binary.LittleEndian, header); err != nil { + return 0, err + } + return w.Writer.Write(p) +} + +func (w *MultiplexWriter) Close() error { return w.Writer.Close() } + +type MultiplexReader struct { + Env *rsyncos.Env + Reader io.Reader +} + +// rsync.h defines IO_BUFFER_SIZE as 32 * 1024, but gokr-rsyncd increases it to +// 256K. Since we use this as the maximum message size, too, we need to at least +// match it. +const ioBufferSize = 256 * 1024 +const maxMessageSize = ioBufferSize + +func (w *MultiplexReader) ReadMsg() (tag uint8, p []byte, err error) { + var header uint32 + if err := binary.Read(w.Reader, binary.LittleEndian, &header); err != nil { + return 0, nil, err + } + + tag = uint8(header>>24) - mplexBase + length := header & 0x00FFFFFF + if length > maxMessageSize { + // NOTE: if you run into this error, one alternative to bumping + // maxMessageSize is to restructure the program to work with i/o buffer + // windowing. + return 0, nil, fmt.Errorf("length %d exceeds max message size (%d)", length, maxMessageSize) + } + p = make([]byte, int(length)) + if _, err := io.ReadFull(w.Reader, p); err != nil { + return 0, nil, err + } + // log.Printf("header=%v (%x), tag=%v, length=%v", header, header, tag, length) + // log.Printf("payload=%x / %q", p, p) + return tag, p, nil +} + +func (w *MultiplexReader) Read(p []byte) (n int, err error) { + tag, payload, err := w.ReadMsg() + if err != nil { + return 0, err + } + switch tag { + case MsgError: + return 0, fmt.Errorf("%s", payload) + case MsgInfo: + w.Env.Logf("info: %s", payload) + // io.ReadFull will call Read again + return 0, nil + case MsgData: + // continues below + default: + return 0, fmt.Errorf("unexpected tag: got %v, want %v", tag, MsgData) + } + if len(p) < len(payload) { + panic(fmt.Sprintf("not enough buffer space! %d < %d", len(p), len(payload))) + } + return copy(p, payload), nil +} + +type Buffer struct { + // buf.Write() never fails, making for a convenient API. + buf bytes.Buffer +} + +func (b *Buffer) WriteByte(data byte) { + binary.Write(&b.buf, binary.LittleEndian, data) +} + +func (b *Buffer) WriteInt32(data int32) { + binary.Write(&b.buf, binary.LittleEndian, data) +} + +func (b *Buffer) WriteInt64(data int64) { + // send as a 32-bit integer if possible + if data <= 0x7FFFFFFF && data >= 0 { + b.WriteInt32(int32(data)) + return + } + // otherwise, send -1 followed by the 64-bit integer + b.WriteInt32(-1) + binary.Write(&b.buf, binary.LittleEndian, data) +} + +func (b *Buffer) WriteString(data string) { + io.WriteString(&b.buf, data) +} + +func (b *Buffer) String() string { + return b.buf.String() +} + +func (b *Buffer) Reset() { + b.buf.Reset() +} + +type Conn struct { + Writer io.WriteCloser + Reader io.ReadCloser +} + +func (c *Conn) Close() error { + wcErr := c.Writer.Close() + rcErr := c.Reader.Close() + if wcErr != nil { + return wcErr + } + return rcErr +} + +func (c *Conn) WriteByte(data byte) error { + return binary.Write(c.Writer, binary.LittleEndian, data) +} + +func (c *Conn) WriteInt32(data int32) error { + return binary.Write(c.Writer, binary.LittleEndian, data) +} + +func (c *Conn) WriteInt64(data int64) error { + // send as a 32-bit integer if possible + if data <= 0x7FFFFFFF && data >= 0 { + return c.WriteInt32(int32(data)) + } + // otherwise, send -1 followed by the 64-bit integer + if err := c.WriteInt32(-1); err != nil { + return err + } + return binary.Write(c.Writer, binary.LittleEndian, data) +} + +func (c *Conn) WriteString(data string) error { + _, err := io.WriteString(c.Writer, data) + return err +} + +func (c *Conn) ReadByte() (byte, error) { + var buf [1]byte + if _, err := io.ReadFull(c.Reader, buf[:]); err != nil { + return 0, err + } + return buf[0], nil +} + +func (c *Conn) ReadInt32() (int32, error) { + var buf [4]byte + if _, err := io.ReadFull(c.Reader, buf[:]); err != nil { + return 0, err + } + return int32(binary.LittleEndian.Uint32(buf[:])), nil +} + +func (c *Conn) ReadInt64() (int64, error) { + { + data, err := c.ReadInt32() + if err != nil { + return 0, err + } + if data != -1 { + // The value was small enough to fit into a 32 bit int, so it was + // transferred directly. + return int64(data), nil + } + // Otherwise, -1 was transmitted, followed by the int64. + } + var data int64 + if err := binary.Read(c.Reader, binary.LittleEndian, &data); err != nil { + return 0, err + } + return data, nil +} + +type CountingReader struct { + R io.ReadCloser + BytesRead int64 +} + +func (r *CountingReader) Read(p []byte) (n int, err error) { + n, err = r.R.Read(p) + r.BytesRead += int64(n) + return n, err +} + +func (r *CountingReader) Close() error { return r.R.Close() } + +type CountingWriter struct { + W io.WriteCloser + BytesWritten int64 +} + +func (w *CountingWriter) Write(p []byte) (n int, err error) { + n, err = w.W.Write(p) + w.BytesWritten += int64(n) + return n, err +} + +func (w *CountingWriter) Close() error { return w.W.Close() } + +func CounterPair(r io.ReadCloser, w io.WriteCloser) (*CountingReader, *CountingWriter) { + crd := &CountingReader{R: r} + cwr := &CountingWriter{W: w} + return crd, cwr +} diff --git a/internal/sender/do.go b/internal/sender/do.go new file mode 100644 index 0000000..8f448ec --- /dev/null +++ b/internal/sender/do.go @@ -0,0 +1,87 @@ +package sender + +import ( + "fmt" + "sort" + + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncstats" + "github.com/runpod/rsync/internal/rsyncwire" +) + +// rsync/main.c:handle_stats +func (st *Transfer) handleStats(crd *rsyncwire.CountingReader, cwr *rsyncwire.CountingWriter, fileList *fileList) error { + if !st.Opts.Server() || !st.Opts.Sender() { + return nil + } + + // send statistics: + // total bytes read (from network connection) + if err := st.Conn.WriteInt64(crd.BytesRead); err != nil { + return err + } + // total bytes written (to network connection) + if err := st.Conn.WriteInt64(cwr.BytesWritten); err != nil { + return err + } + // total size of files + if err := st.Conn.WriteInt64(fileList.TotalSize); err != nil { + return err + } + return nil +} + +// rsync/main.c:client_run am_sender +func (st *Transfer) Do(crd *rsyncwire.CountingReader, cwr *rsyncwire.CountingWriter, modPath string, paths []string, exclusionList *filterRuleList) (*rsyncstats.TransferStats, error) { + if exclusionList == nil { + exclusionList = &filterRuleList{} + } + + // “Update exchange” as per + // https://github.com/kristapsdz/openrsync/blob/master/rsync.5 + + // send file list + st.Logger.Printf("SendFileList(modPath=%q, paths=%q)", modPath, paths) + fileList, err := st.SendFileList(modPath, paths, exclusionList) + if err != nil { + return nil, err + } + defer fileList.Close() + + if st.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 3) { + st.Logger.Printf("file list sent") + } + + // Sort the file list. The client sorts, so we need to sort, too (in the + // same way!), otherwise our indices do not match what the client will + // request. + sort.Slice(fileList.Files, func(i, j int) bool { + return fileList.Files[i].Wpath < fileList.Files[j].Wpath + }) + + if err := st.SendFiles(fileList); err != nil { + return nil, err + } + + if err := st.handleStats(crd, cwr, fileList); err != nil { + return nil, err + } + + if st.Opts.DebugGTE(rsyncopts.DEBUG_PROTO, 1) { + st.Logger.Printf("reading final int32") + } + + finish, err := st.Conn.ReadInt32() + if err != nil { + return nil, err + } + if finish != -1 { + return nil, fmt.Errorf("protocol error: expected final -1, got %d", finish) + } + + return &rsyncstats.TransferStats{ + Read: crd.BytesRead, + Written: cwr.BytesWritten, + Size: fileList.TotalSize, + }, nil +} diff --git a/internal/sender/exclude.go b/internal/sender/exclude.go new file mode 100644 index 0000000..dc1a0a9 --- /dev/null +++ b/internal/sender/exclude.go @@ -0,0 +1,109 @@ +package sender + +import ( + "io" + "path/filepath" + "strings" + + "github.com/runpod/rsync/internal/rsyncwire" +) + +type filterRuleList struct { + Filters []*filterRule +} + +// exclude.c:add_rule +func (l *filterRuleList) addRule(fr *filterRule) { + if strings.HasSuffix(fr.pattern, "/") { + fr.flag |= filtruleDirectory + fr.pattern = strings.TrimSuffix(fr.pattern, "/") + } + if strings.ContainsFunc(fr.pattern, func(r rune) bool { + return r == '*' || r == '[' || r == '?' + }) { + fr.flag |= filtruleWild + } + l.Filters = append(l.Filters, fr) +} + +// exclude.c:check_filter +func (l *filterRuleList) matches(name string) bool { + for _, fr := range l.Filters { + if fr.matches(name) { + return true + } + } + return false +} + +// exclude.c:recv_filter_list +func RecvFilterList(c *rsyncwire.Conn) (*filterRuleList, error) { + var l filterRuleList + const exclusionListEnd = 0 + for { + length, err := c.ReadInt32() + if err != nil { + return nil, err + } + if length == exclusionListEnd { + break + } + line := make([]byte, length) + if _, err := io.ReadFull(c.Reader, line); err != nil { + return nil, err + } + fr, err := parseFilter(string(line)) + if err != nil { + return nil, err + } + l.addRule(fr) + } + return &l, nil +} + +const ( + filtruleInclude = 1 << iota + filtruleClearList + filtruleDirectory + filtruleWild +) + +type filterRule struct { + flag int + pattern string +} + +// exclude.c:rule_matches +func (fr *filterRule) matches(name string) bool { + if fr.flag&filtruleWild != 0 { + panic("wildcard filter rules not yet implemented") + } + if !strings.ContainsRune(fr.pattern, '/') && + fr.flag&filtruleWild == 0 { + name = filepath.Base(name) + } + return fr.pattern == name +} + +// exclude.c:parse_filter_str / exclude.c:parse_rule_tok +func parseFilter(line string) (*filterRule, error) { + rule := new(filterRule) + + // We only support what rsync calls XFLG_OLD_PREFIXES + if strings.HasPrefix(line, "- ") { + // clear include flag + rule.flag &= ^filtruleInclude + line = strings.TrimPrefix(line, "- ") + } else if strings.HasPrefix(line, "+ ") { + // set include flag + rule.flag |= filtruleInclude + line = strings.TrimPrefix(line, "+ ") + } else if strings.HasPrefix(line, "!") { + // set clear_list flag + rule.flag |= filtruleClearList + } + + rule.pattern = line + + return rule, nil +} diff --git a/internal/sender/fileio.go b/internal/sender/fileio.go new file mode 100644 index 0000000..f46d285 --- /dev/null +++ b/internal/sender/fileio.go @@ -0,0 +1,112 @@ +package sender + +import ( + "fmt" + "io" +) + +// rsync.h:map_struct +type mapStruct struct { + fileSize int64 // file size (from stat) + pOffset int64 // window start + pFdOffset int64 // offset of cursor in fd ala lseek + window []byte + pSize int64 // largest window we allocated + pLen int64 // latest (rounded) window size + defWindowSize int64 // default window size + f File // file handle (fs.File + io.Seeker) + err error // first read error +} + +const alignBoundary = 1024 + +func alignedLength(l int64) int64 { + return ((l - 1) | (alignBoundary - 1)) + 1 +} + +func alignedOvershoot(off int64) int64 { + return off & (alignBoundary - 1) +} + +func mapFile(f File, len int64, readSize int32, blkSize int32) *mapStruct { + if blkSize > 0 && readSize%blkSize != 0 { + readSize += blkSize - (readSize % blkSize) + } + return &mapStruct{ + fileSize: len, + defWindowSize: alignedLength(int64(readSize)), + f: f, + } +} + +func (ms *mapStruct) ptr(offset int64, l int32) ([]byte, error) { + //log.Printf("ptr(offset=%d, l=%d)", offset, l) + len := int64(l) + if len == 0 { + return nil, nil + } + if len < 0 { + return nil, fmt.Errorf("invalid length: %d < 0", len) + } + + if offset >= ms.pOffset && offset+int64(len) <= ms.pOffset+int64(ms.pLen) { + //log.Printf("-> already available") + // region already available + off := offset - ms.pOffset + return ms.window[off : off+int64(len)], nil + } + + alignFudge := alignedOvershoot(offset) + windowStart := offset - alignFudge + windowSize := int64(ms.defWindowSize) + if windowStart+windowSize > ms.fileSize { + windowSize = ms.fileSize - windowStart + } + if windowSize < len+alignFudge { + windowSize = alignedLength(len + alignFudge) + } + if windowSize > ms.pSize { + win := make([]byte, windowSize) + copy(win, ms.window) + ms.window = win + ms.pSize = windowSize + } + readStart := windowStart + readSize := windowSize + readOffset := int64(0) + + //log.Printf("windowSize: %d, ms=%+v", windowSize, ms) + if windowStart >= ms.pOffset && windowStart < ms.pOffset+ms.pLen && + windowStart+windowSize >= ms.pOffset+ms.pLen { + readStart = ms.pOffset + ms.pLen + readOffset = readStart - windowStart + readSize = windowSize - readOffset + off := ms.pLen - readOffset + copy(ms.window[:], ms.window[off:off+readOffset]) + } + if readSize <= 0 { + return nil, fmt.Errorf("invalid readSize: %d <= 0", readSize) + } + if ms.pFdOffset != readStart { + if _, err := ms.f.Seek(readStart, io.SeekStart); err != nil { + return nil, fmt.Errorf("seek error: %v", err) + } + ms.pFdOffset = readStart + } + ms.pOffset = windowStart + ms.pLen = windowSize + //log.Printf("-> reading %d bytes from %d into buffer at offset=%d", readSize, readStart, readOffset) + for readSize > 0 { + n, err := ms.f.Read(ms.window[readOffset : readOffset+readSize]) + if err != nil { + ms.err = err + // TODO: zero the buffer, file has changed mid-transfer + return nil, fmt.Errorf("file has changed mid-transfer") + break + } + ms.pFdOffset += int64(n) + readOffset += int64(n) + readSize -= int64(n) + } + return ms.window[alignFudge : alignFudge+len], nil +} diff --git a/internal/sender/flist.go b/internal/sender/flist.go new file mode 100644 index 0000000..06e3c1b --- /dev/null +++ b/internal/sender/flist.go @@ -0,0 +1,469 @@ +package sender + +import ( + "fmt" + "io/fs" + "os" + "os/user" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncchecksum" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncwire" +) + +type file struct { + source FileSource + path string + Wpath string + regular bool + + // fields below are used by the receiver (TODO: unify) + Name string + Length int64 + ModTime time.Time + Mode int32 + Uid int32 + Gid int32 + LinkTarget string + Rdev int32 +} + +type fileList struct { + TotalSize int64 + Files []file + Sources []FileSource +} + +// A fileList must not be used after calling Close(). +func (fl *fileList) Close() { + for _, source := range fl.Sources { + source.Close() + } + fl.Sources = nil +} + +// rsync/rsync.h defines chunkSize as 32 * 1024, which we match. +// +// Other parts of rsync have subtle dependencies on the chunkSize. +// For example, (*sender.Transfer).hashSearch declares a window +// of at least 256 KB and then reads from that window, requesting +// 2*chunkSize+alignFudge, which must not exceed the window. +// So raising chunkSize to 128 KB or 256 KB breaks transfers +// with "file has changed mid-transfer" (issue #53). +const chunkSize = 32 * 1024 + +var ( + lookupOnce sync.Once + lookupGroupOnce sync.Once +) + +// getStrip operates on wire paths (slash space). +func getStrip(requested string) string { + if requested == "/" { + return "" + } + if strings.HasSuffix(requested, "/") { + return strings.TrimPrefix(path.Clean(requested), "/") + "/" + } + return "" +} + +type scopedWalker struct { + st *Transfer + ioError func(err error) + conn *rsyncwire.Conn + fec *rsyncwire.Buffer + excl *filterRuleList + uidMap map[int32]string + gidMap map[int32]string + fileList *fileList + source FileSource + localDir string + requested string + strip string + subdir string + prefix string +} + +func (s *scopedWalker) walk() error { + if s.source == nil { + fi, err := os.Lstat(s.localDir) + if err != nil { + s.st.Logger.Printf(" Lstat(localDir=%q): %v", s.localDir, err) + return fmt.Errorf("i/o error: requested module path is not accessible") + } + if fi.IsDir() { + root, err := os.OpenRoot(s.localDir) + if err != nil { + s.st.Logger.Printf(" OpenRoot(localDir=%q): %v", s.localDir, err) + return fmt.Errorf("i/o error: requested module path is not accessible") + } + s.source = newOSRootSource(root) + } else { + s.source = newSingleFileSource(s.localDir, fi) + } + s.fileList.Sources = append(s.fileList.Sources, s.source) + } + if s.subdir != "." { + sub, err := newSubSource(s.source, s.subdir) + if err != nil { + s.st.Logger.Printf(" newSubSource(subdir=%q): %v", s.subdir, err) + return fmt.Errorf("i/o error: requested module path is not accessible") + } + s.source = sub + } + + rootname := s.requested + // fs.WalkDir(root.FS(), …) does not accept absolute paths, + // so make them relative by prepending a . + if strings.HasPrefix(rootname, "/") { + rootname = "." + rootname + } + if err := fs.WalkDir(s.source.FS(), path.Clean(rootname), s.walkFn); err != nil { + return err + } + return nil +} + +func (s *scopedWalker) walkFn(path string, d fs.DirEntry, err error) error { + logger := s.st.Logger // for convenience + opts := s.st.Opts // for convenience + if opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + logger.Printf("filepath.WalkFn(path=%s)", path) + } + var info fs.FileInfo + if err == nil { + info, err = d.Info() + } + if err != nil { + // set the I/O error flag, but keep walking + s.ioError(err) + return nil + } + + if opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + logger.Printf("isDir=%v, xferDirs=%v", info.Mode().IsDir(), opts.XferDirs()) + } + if info.Mode().IsDir() && opts.XferDirs() == 0 { + logger.Printf("skipping directory %s", path) + return filepath.SkipDir + } + + // Only ever transmit long names, like openrsync + flags := byte(rsync.XMIT_LONG_NAME) + + name := path + if s.strip != "" { + if path+"/" == s.strip { + // Transmit the top directory as ., like tridge rsync. + name = "." + } else { + name = strings.TrimPrefix(name, s.strip) + } + } + if s.prefix != "" { + if path == "." { + name = s.prefix + } else { + name = s.prefix + "/" + path + } + } + if opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + logger.Printf("Trim(path=%q) = %q", path, name) + } + if path == "." && s.prefix == "" { + flags |= rsync.XMIT_TOP_DIR + } + // st.logger.Printf("flags for %q: %v", name, flags) + + if s.excl.matches(name) { + return filepath.SkipDir + } + + s.fileList.Files = append(s.fileList.Files, file{ + source: s.source, + path: path, + regular: info.Mode().IsRegular(), + Wpath: name, + Length: info.Size(), + }) + + s.fec.Reset() + + // 1. status byte (integer) + s.fec.WriteByte(flags) + + // 2. inherited filename length (optional, byte) + // 3. filename length (integer or byte) + s.fec.WriteInt32(int32(len(name))) + + // 4. file (byte array) + s.fec.WriteString(name) + + // 5. file length (long) + size := info.Size() + if info.Mode().IsDir() { + // tmpfs returns non-4K sizes for directories. Override with + // 4096 to make the tests succeed regardless of the /tmp file + // system type. + size = 4096 + } + s.fec.WriteInt64(size) + + s.fileList.TotalSize += size + + // 6. file modification time (optional, integer) + // TODO: this will overflow in 2038! :( + s.fec.WriteInt32(int32(info.ModTime().Unix())) + + // 7. file mode (optional, mode_t, integer) + mode := int32(info.Mode() & os.ModePerm) + isDev := false + isSpecial := false + if info.Mode().IsDir() { + mode |= rsync.S_IFDIR + } else if info.Mode().IsRegular() { + mode |= rsync.S_IFREG + } else if info.Mode().Type()&os.ModeSymlink != 0 { + mode |= rsync.S_IFLNK + // TODO: skip symlink if PreserveSymlinks is not set + } + + if info.Mode().Type()&os.ModeCharDevice != 0 { + mode |= rsync.S_IFCHR + isDev = true + } else if info.Mode().Type()&os.ModeDevice != 0 { + mode |= rsync.S_IFBLK + isDev = true + } + + if info.Mode().Type()&os.ModeNamedPipe != 0 { + mode |= rsync.S_IFIFO + isSpecial = true + } + + if info.Mode().Type()&os.ModeSocket != 0 { + mode |= rsync.S_IFSOCK + isSpecial = true + } + + s.fec.WriteInt32(mode) + + if opts.PreserveUid() { + uid, ok := uidFromFileInfo(info) + if ok { + if _, ok := s.uidMap[uid]; !ok && uid != 0 { + u, err := user.LookupId(strconv.Itoa(int(uid))) + if err != nil { + lookupOnce.Do(func() { + logger.Printf("lookup(%d) = %v", uid, err) + }) + } else { + s.uidMap[uid] = u.Username + } + } + } + // 8. if -o, the user id (integer) + s.fec.WriteInt32(uid) + } + + if opts.PreserveGid() { + gid, ok := gidFromFileInfo(info) + if ok { + if _, ok := s.gidMap[gid]; !ok && gid != 0 { + g, err := user.LookupGroupId(strconv.Itoa(int(gid))) + if err != nil { + lookupGroupOnce.Do(func() { + logger.Printf("lookupgroup(%d) = %v", gid, err) + }) + } else { + s.gidMap[gid] = g.Name + } + } + } + // 9. if -g, the group id (integer) + s.fec.WriteInt32(gid) + } + + if (opts.PreserveDevices() && isDev) || + (opts.PreserveSpecials() && isSpecial) { + // 10. if a special file and -D, the device “rdev” type (integer) + rdev, _ := rdevFromFileInfo(info) + s.fec.WriteInt32(rdev) + } + + if opts.PreserveLinks() && info.Mode().Type()&os.ModeSymlink != 0 { + // 11. if a symbolic link and -l, the link target's length (integer) + // 12. if a symbolic link and -l, the link target (byte array) + + target, err := s.source.Readlink(path) + if err != nil { + return err // TODO + } + s.fec.WriteInt32(int32(len(target))) + s.fec.WriteString(target) + } + + if opts.AlwaysChecksum() { + var emptyChecksum [rsyncchecksum.Size]byte + checksum := emptyChecksum[:] + if info.Mode().IsRegular() { + f, err := s.source.Open(path) + if err != nil { + return err + } + checksum, err = rsyncchecksum.ReaderChecksum(f) + f.Close() + if err != nil { + return err + } + } else { + // send empty md4 checksum + } + s.fec.WriteString(string(checksum)) + } + + s.conn.WriteString(s.fec.String()) + + // The status byte may consist of the following bits and determines which of the optional fields are transmitted. + + // 0x01 A top-level directory. (Only applies to directory files.) If specified, the matching local directory is for deletions. + // 0x02 Do not send the file mode: it is a repeat of the last file's mode. + // 0x08 Like 0x02, but for the user id. + // 0x10 Like 0x02, but for the group id. + // 0x20 Inherit some of the prior file name. Enables the inherited filename length transmission. + // 0x40 Use full integer length for file name. Otherwise, use only the byte length. + // 0x80 Do not send the file modification time: it is a repeat of the last file's. + + // If the status byte is zero, the file-list has terminated. + + if info.Mode().IsDir() && !opts.Recurse() { + return filepath.SkipDir + } + + return nil +} + +// rsync/flist.c:send_file_list +func (st *Transfer) SendFileList(localDir string, paths []string, excl *filterRuleList) (*fileList, error) { + var fileList fileList + fec := &rsyncwire.Buffer{} + + uidMap := make(map[int32]string) + gidMap := make(map[int32]string) + + // TODO: flush in between to keep the pipes filled when traversal takes long + + // TODO: handle info == nil case (permission denied?): should set an i/o + // error flag, but traversal should continue + + st.Logger.Printf("building file list") + if st.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + st.Logger.Printf("sendFileList()") + } + ioErrors := int32(0) + + ioError := func(err error) { + if os.IsNotExist(err) { + st.Logger.Printf("file vanished: %v", err) + } else { + st.Logger.Printf("lstat: %v", err) + } + ioErrors = 1 + } + + for _, requested := range paths { + subdir := "." + prefix := "" + local := localDir + if local == rsync.FileSystemRoot { + // Implicit module (/) and absolute requested path (/tmp/foo/), + // turn the path into the local directory and request /. + local = filepath.Clean(requested) + if strings.HasSuffix(requested, "/") { + + } else { + prefix = filepath.Base(requested) + } + requested = "/" + } else if !strings.HasSuffix(requested, "/") { + st.Logger.Printf(" handling requested=%q", requested) + clean := path.Clean(strings.TrimPrefix(requested, "/")) + subdir = path.Dir(clean) + requested = path.Base(clean) + st.Logger.Printf(" -> subdir=%q, requested=%q", subdir, requested) + } + + if st.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + st.Logger.Printf(" path %q (local dir %q)", requested, local) + } + // st.Logger.Printf("getRootStrip(requested=%q, localDir=%q", requested, localDir) + strip := getStrip(requested) + // st.Logger.Printf("root=%q, strip=%q", root, strip) + if st.Opts.DebugGTE(rsyncopts.DEBUG_FLIST, 1) { + st.Logger.Printf(" fs.Walk(%q, %q), strip=%q", local, requested) + } + + sw := &scopedWalker{ + st: st, + conn: st.Conn, + fec: fec, + excl: excl, + uidMap: uidMap, + gidMap: gidMap, + fileList: &fileList, + source: st.Source, + ioError: ioError, + localDir: local, + requested: requested, + strip: strip, + subdir: subdir, + prefix: prefix, + } + if err := sw.walk(); err != nil { + return nil, err + } + } + + if st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + st.Logger.Printf("%d files to consider", len(fileList.Files)) + } + + fec.Reset() + + const endOfFileList = 0 + fec.WriteByte(endOfFileList) + + const endOfSet = 0 + if st.Opts.PreserveUid() { + for uid, name := range uidMap { + fec.WriteInt32(uid) + fec.WriteByte(byte(len(name))) + fec.WriteString(name) + } + fec.WriteInt32(endOfSet) + } + if st.Opts.PreserveGid() { + for gid, name := range gidMap { + fec.WriteInt32(gid) + fec.WriteByte(byte(len(name))) + fec.WriteString(name) + } + fec.WriteInt32(endOfSet) + } + + fec.WriteInt32(ioErrors) + + if err := st.Conn.WriteString(fec.String()); err != nil { + return nil, err + } + + return &fileList, nil +} diff --git a/internal/sender/flist_test.go b/internal/sender/flist_test.go new file mode 100644 index 0000000..731ea30 --- /dev/null +++ b/internal/sender/flist_test.go @@ -0,0 +1,38 @@ +package sender + +import "testing" + +func TestPath(t *testing.T) { + for _, tt := range []struct { + requested string + wantRoot string + wantStrip string + }{ + { + requested: "/", // sent by client + wantStrip: "", + }, + + { + requested: "tr/man5", // sent by client + wantStrip: "", + }, + + { + requested: "tr/", // sent by client + wantStrip: "tr/", + }, + + { // client started with src=/usr/share/man + requested: "", // sent by client + wantStrip: "", + }, + } { + t.Run("requested="+tt.requested, func(t *testing.T) { + gotStrip := getStrip(tt.requested) + if gotStrip != tt.wantStrip { + t.Errorf("unexpected strip: got %q, want %q", gotStrip, tt.wantStrip) + } + }) + } +} diff --git a/internal/sender/match.go b/internal/sender/match.go new file mode 100644 index 0000000..f9bb5cc --- /dev/null +++ b/internal/sender/match.go @@ -0,0 +1,282 @@ +package sender + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash" + + "github.com/mmcloughlin/md4" + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncchecksum" + "github.com/runpod/rsync/internal/rsyncopts" +) + +type target struct { + index int32 + tag uint16 +} + +// rsync/match.c:hash_search +func (st *Transfer) hashSearch(targets []target, tagTable map[uint16]int, head rsync.SumHead, fileIndex int32, fl file) error { + st.Logger.Printf("hashSearch(path=%s, len(sums)=%d)", fl.path, len(head.Sums)) + f, err := fl.source.Open(fl.path) + if err != nil { + return err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return err + } + + readSize := max(3*head.BlockLength, 256*1024) + ms := mapFile(f, fi.Size(), readSize, head.BlockLength) + + if err := st.Conn.WriteInt32(fileIndex); err != nil { + return err + } + + if err := head.WriteTo(st.Conn); err != nil { + return err + } + + if !st.Opts.Server() && + st.Opts.InfoGTE(rsyncopts.INFO_NAME, 1) && + st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + fmt.Fprintln(st.Env.Stdout, fl.path) + } + + // sum_init() + h := md4.New() + binary.Write(h, binary.LittleEndian, st.Seed) + + // The following quotes are citations from + // https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The + // signature search algorithm (PDF page 64). + + // “Once the sorted signature table and the index table have been formed the + // signature search process can begin. For each byte offset in a_i the fast + // signature is computed, along with the 16 bit hash of the fast + // signature. The 16 bit hash is then used to lookup the signature index, + // giving the index in the signature table of the first fast signature with + // that hash.” + + var k int + var sum uint32 + var s1, s2 uint32 + var offset int64 + end := fi.Size() + 1 - head.Sums[len(head.Sums)-1].Len + st.Logger.Printf("last block len=%d, end=%d", head.Sums[len(head.Sums)-1].Len, end) + + readChunk := func() error { + k = int(head.BlockLength) + if remaining := int(fi.Size() - offset); remaining < k { + k = remaining + } + + chunk, err := ms.ptr(offset, int32(k)) + if err != nil { + return err + } + sum = rsyncchecksum.Checksum1(chunk) + s1 = uint32(sum & 0xFFFF) + s2 = uint32(sum >> 16) + return nil + } + if err := readChunk(); err != nil { + return err + } + + tagHits := 0 +Outer: + for { + tag := rsyncchecksum.Tag2(uint16(s1), uint16(s2)) + var sum2 []byte + doneCsum2 := false + j, ok := tagTable[tag] + if ok { + // “A linear search is then performed through the signature table, stopping + // when an entry is found with a 16 bit hash which doesn’t match. For each + // entry the current 32 bit fast signature is compared to the entry in the + // signature table, and if that matches then the full 128 bit strong + // signature is computed at the current byte offset and compared to the + // strong signature in the signature table” + sum = (uint32(s1) & 0xFFFF) | (uint32(s2) << 16) + tagHits++ + for ; j < int(head.ChecksumCount) && targets[j].tag == tag; j++ { + i := targets[j].index + if sum != head.Sums[i].Sum1 { + continue + } + + l := int64(head.BlockLength) + if v := fi.Size() - offset; v < l { + l = v + } + if l != head.Sums[i].Len { + continue + } + + // st.logger.Printf("potential match at %d target=%d %d sum=%08x", offset, j, i, sum) + + if !doneCsum2 { + buf, err := ms.ptr(offset, int32(l)) + if err != nil { + return err + } + sum2 = rsyncchecksum.Checksum2(st.Seed, buf[:]) + doneCsum2 = true + } + + if local, remote := sum2[:head.ChecksumLength], head.Sums[i].Sum2[:head.ChecksumLength]; !bytes.Equal(local, remote) { + st.Logger.Printf("false alarm: local %x, remote %x", local, remote) + //falseAlarms++ + continue + } + + // TODO(optimization): tridge rsync locates adjacent matches + // here for better run-length encoding, but I’m not sure where + // (if at all) we currently use run-length encoding: + // https://github.com/WayneD/rsync/commit/923fa978088f4c044eec528d9472962d9c9d13c3 + + // “If the strong signature is found to match then A emits a + // token telling B that a match was found and which block in bi + // was matched12. The search then continues at the byte after + // the matching block.” + + if err := st.matched(h, ms, head, offset, i); err != nil { + return err + } + + // rsync doesn’t read the next chunk (offset+sums[i].len), + // rsync starts reading one byte before the next chunk + // (offset+sums[i].len-1), because the code path starting at + // “null_tag” removes the chunk’s first byte and adds the + // next byte after the chunk. + offset += head.Sums[i].Len - 1 + if err := readChunk(); err != nil { + return fmt.Errorf("readChunk: %v", err) + } + + if offset >= end { + break Outer + } + + break + } + } + + // Update the rolling checksum by removing the oldest byte (update[0]) + // and adding the newest byte (update[k]). + backup := max(offset-st.lastMatch, 0) + + more := offset+int64(k) < fi.Size() + mmore := int64(0) + if more { + mmore = 1 + } + update, err := ms.ptr(offset-backup, int32(int64(k)+mmore+backup)) + if err != nil { + return err + } + update = update[backup:] + + s1 -= rsyncchecksum.SignExtend(update[0]) + s2 -= uint32(k) * rsyncchecksum.SignExtend(update[0]) + + if more { + s1 += rsyncchecksum.SignExtend(update[k]) + s2 += s1 + } else { + k-- + } + s1 = uint32(uint16(s1)) + s2 = uint32(uint16(s2)) + + if backup >= int64(head.BlockLength)+chunkSize && end-offset > chunkSize { + // Prevent offset-st.lastMatch from growing too large by flushing + // intermediate chunks. + if err := st.matched(h, ms, head, offset-int64(head.BlockLength), -2); err != nil { + return err + } + } + + offset++ + if offset >= end { + break + } + } + + if err := st.matched(h, ms, head, fi.Size(), -1); err != nil { + return err + } + + if st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + st.Progress.Show(uint64(offset), true) + } + + { + sum := h.Sum(nil) + st.Logger.Printf("sum: %x (len = %d)", sum, len(sum)) + if _, err := st.Conn.Writer.Write(sum); err != nil { + return err + } + } + + return nil + +} + +// rsync/match.c:matched +func (st *Transfer) matched(h hash.Hash, ms *mapStruct, head rsync.SumHead, offset int64, i int32) error { + n := offset - st.lastMatch + + transmitAccumulated := i < 0 + + // if !transmitAccumulated { + // st.logger.Printf("match at offset=%d last_match=%d i=%d len=%d n=%d", + // offset, st.lastMatch, i, head.Sums[i].Len, n) + // } else { + // st.logger.Printf("transmit accumulated at offset=%d", offset) + // } + + /* FIXME: this is not used + l := int64(0) + if !transmitAccumulated { + l = head.Sums[i].Len + } + */ + + if err := st.sendToken(ms, i, st.lastMatch, n); err != nil { + return fmt.Errorf("sendToken: %v", err) + } + // TODO: data_transfer += n; + + if !transmitAccumulated { + // stats.matched_data += s->sums[i].len; + n += head.Sums[i].Len + } + + for j := int64(0); j < n; j += chunkSize { + n1 := min(int64(chunkSize), n-j) + chunk, err := ms.ptr(st.lastMatch+j, int32(n1)) + if err != nil { + return err + } + h.Write(chunk) + } + + if !transmitAccumulated { + st.lastMatch = offset + head.Sums[i].Len + } else { + st.lastMatch = offset + } + + if st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + st.Progress.MaybeShow(uint64(offset), false) + } + + return nil +} diff --git a/internal/sender/sender.go b/internal/sender/sender.go new file mode 100644 index 0000000..1749567 --- /dev/null +++ b/internal/sender/sender.go @@ -0,0 +1,250 @@ +package sender + +import ( + "encoding/binary" + "fmt" + "io" + "os" + "sort" + + "github.com/mmcloughlin/md4" + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncchecksum" + "github.com/runpod/rsync/internal/rsynccommon" + "github.com/runpod/rsync/internal/rsyncopts" + "golang.org/x/sync/errgroup" +) + +// rsync/sender.c:send_files() +func (st *Transfer) SendFiles(fileList *fileList) error { + phase := 0 + for { + // receive data about receiver’s copy of the file list contents (not + // ordered) + // see (*rsync.Receiver).Generator() + fileIndex, err := st.Conn.ReadInt32() + if err != nil { + return err + } + if fileIndex == -1 { + if phase == 0 { + phase++ + // acknowledge phase change by sending -1 + if err := st.Conn.WriteInt32(-1); err != nil { + return err + } + continue + } + break + } + + if st.Opts.DryRun() { + if err := st.Conn.WriteInt32(fileIndex); err != nil { + return err + } + continue + } + + fl := fileList.Files[fileIndex] + st.Progress.Reset(uint64(fl.Length)) + + head, err := st.receiveSums() + if err != nil { + return err + } + + // The following quotes are citations from + // https://www.samba.org/~tridge/phd_thesis.pdf, section 3.2.6 The + // signature search algorithm (PDF page 64). + + // rsync/match.c:build_hash_table + targets := make([]target, len(head.Sums)) + tagTable := make(map[uint16]int) // TODO: or int32 more specifically? + { + // “The first step in the algorithm is to sort the received + // signatures by a 16 bit hash of the fast signature.” + for idx, sum := range head.Sums { + targets[idx] = target{ + index: int32(idx), + tag: rsyncchecksum.Tag(sum.Sum1), + } + } + sort.Slice(targets, func(i, j int) bool { + return targets[i].tag < targets[j].tag + }) + + // “A 16 bit index table is then formed which takes a 16 bit hash + // value and gives an index into the sorted signature table which + // points to the first entry in the table which has a matching + // hash.” + for idx := len(head.Sums) - 1; idx >= 0; idx-- { + tagTable[targets[idx].tag] = idx + } + } + + st.lastMatch = 0 + if len(head.Sums) == 0 { + // fast path: send the whole file + err = st.sendFile(fileIndex, fl) + } else { + err = st.hashSearch(targets, tagTable, head, fileIndex, fl) + } + if err != nil { + if _, ok := err.(*os.PathError); ok { + // OpenFile() failed. Log the error (server side only) and + // proceed. Only starting with protocol 30, an I/O error flag is + // sent after the file transfer phase. + if os.IsNotExist(err) { + st.Logger.Printf("file has vanished: %s", fl.path) + } else { + st.Logger.Printf("sendFiles: %v", err) + } + continue + } else { + return err + } + } + } + + // phase done + if err := st.Conn.WriteInt32(-1); err != nil { + return err + } + + return nil +} + +// rsync/sender.c:receive_sums() +func (st *Transfer) receiveSums() (rsync.SumHead, error) { + var head rsync.SumHead + if err := head.ReadFrom(st.Conn); err != nil { + return head, err + } + var offset int64 + head.Sums = make([]rsync.SumBuf, int(head.ChecksumCount)) + for i := int32(0); i < head.ChecksumCount; i++ { + shortChecksum, err := st.Conn.ReadInt32() + if err != nil { + return head, err + } + sb := rsync.SumBuf{ + Index: i, + Offset: offset, + Sum1: uint32(shortChecksum), + } + if i == head.ChecksumCount-1 && head.RemainderLength != 0 { + sb.Len = int64(head.RemainderLength) + } else { + sb.Len = int64(head.BlockLength) + } + offset += sb.Len + n, err := io.ReadFull(st.Conn.Reader, sb.Sum2[:head.ChecksumLength]) + if err != nil { + return head, err + } + _ = n + // st.logger.Printf("chunk[%d] len=%d offset=%.0f sum1=%08x, sum2=%x", + // i, sb.len, float64(sb.offset), sb.sum1, sb.sum2[:n]) + head.Sums[i] = sb + } + return head, nil +} + +func (st *Transfer) sendFile(fileIndex int32, fl file) error { + // rsync/rsync.h defines chunkSize as 32 * 1024, but increasing it to 256K + // increases throughput with “tridge” rsync as client by 50 Mbit/s. + const chunkSize = 256 * 1024 + + f, err := fl.source.Open(fl.path) + if err != nil { + return err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return err + } + + if err := st.Conn.WriteInt32(fileIndex); err != nil { + return err + } + + sh := rsynccommon.SumSizesSqroot(fi.Size()) + if err := sh.WriteTo(st.Conn); err != nil { + return err + } + + if !st.Opts.Server() && + st.Opts.InfoGTE(rsyncopts.INFO_NAME, 1) && + st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + fmt.Fprintln(st.Env.Stdout, fl.path) + } + + h := md4.New() + binary.Write(h, binary.LittleEndian, st.Seed) + + // Calculate the md4 hash in a goroutine. + // + // This allows an rsync connection to benefit from more than 1 core! + // + // We calculate the hash by opening the same file again and reading + // independently. This keeps the hot loop below focused on shoveling data + // into the network socket as quickly as possible. + var eg errgroup.Group + eg.Go(func() error { + f, err := fl.source.Open(fl.path) + if err != nil { + return err + } + defer f.Close() + var buf [chunkSize]byte + if _, err := io.CopyBuffer(h, f, buf[:]); err != nil { + return err + } + return nil + }) + + offset := 0 + buf := make([]byte, chunkSize) + for { + if st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + st.Progress.MaybeShow(uint64(offset), false) + } + n, err := f.Read(buf) + if err != nil { + if err == io.EOF { + break + } + return err + } + chunk := buf[:n] + // chunk size (“rawtok” variable in openrsync) + if err := st.Conn.WriteInt32(int32(len(chunk))); err != nil { + return err + } + n, err = st.Conn.Writer.Write(chunk) + if err != nil { + return err + } + offset += n + } + if st.Opts.InfoGTE(rsyncopts.INFO_PROGRESS, 1) { + st.Progress.Show(uint64(offset), true) + } + // transfer finished: + if err := st.Conn.WriteInt32(0); err != nil { + return err + } + + // whole file long checksum (16 bytes) + if err := eg.Wait(); err != nil { + return err + } + sum := h.Sum(nil) + // st.logger.Printf("sum: %x (len = %d)", sum, len(sum)) + if _, err := st.Conn.Writer.Write(sum); err != nil { + return err + } + return nil +} diff --git a/internal/sender/source.go b/internal/sender/source.go new file mode 100644 index 0000000..31bd83f --- /dev/null +++ b/internal/sender/source.go @@ -0,0 +1,140 @@ +package sender + +import ( + "fmt" + "io" + "io/fs" + "os" + "path" +) + +// FileSource is the interface which the gokrazy rsync sender uses +// to discover and read files. This allows working with rsync modules +// that are backed by an actual file system (*os.Root) or fs.FS. +type FileSource interface { + // FS returns the underlying fs.FS for use with fs.WalkDir. + FS() fs.FS + + // Open opens a file. Returns error if file does not implement io.Seeker. + Open(name string) (File, error) + + // Readlink reads a symlink target. Needs fs.ReadLinkFS. + Readlink(name string) (string, error) + + Close() error +} + +type File interface { + fs.File + io.Seeker +} + +type osRootSource struct { + root *os.Root +} + +func newOSRootSource(root *os.Root) FileSource { + return &osRootSource{root: root} +} + +func (s *osRootSource) FS() fs.FS { return s.root.FS() } +func (s *osRootSource) Open(name string) (File, error) { return s.root.Open(name) } +func (s *osRootSource) Readlink(name string) (string, error) { return s.root.Readlink(name) } +func (s *osRootSource) Close() error { return s.root.Close() } + +// singleFileSource implements FileSource. +type singleFileSource struct { + path string + info fs.FileInfo +} + +func newSingleFileSource(path string, info fs.FileInfo) *singleFileSource { + return &singleFileSource{ + path: path, + info: info, + } +} + +func (s *singleFileSource) FS() fs.FS { return &singleFileFS{s} } +func (s *singleFileSource) Open(_ string) (File, error) { return os.Open(s.path) } +func (s *singleFileSource) Readlink(_ string) (string, error) { return os.Readlink(s.path) } +func (s *singleFileSource) Close() error { return nil } + +type singleFileFS struct { + sfs *singleFileSource +} + +func (f *singleFileFS) Open(name string) (fs.File, error) { return os.Open(f.sfs.path) } +func (f *singleFileFS) Stat(name string) (fs.FileInfo, error) { return f.sfs.info, nil } + +// fsSource wraps an fs.FS to implement FileSource. +type fsSource struct { + fsys fs.FS +} + +// NewFSSource creates a FileSource from an fs.FS. +// +// Files returned by the fs.FS must implement io.Seeker, +// otherwise Open will fail. +// +// The fs.FS should implement ReadLinkFS, +// otherwise working with symlinks will fail. +func NewFSSource(fsys fs.FS) FileSource { + return &fsSource{fsys: fsys} +} + +func (s *fsSource) FS() fs.FS { return s.fsys } + +func (s *fsSource) Open(name string) (File, error) { + f, err := s.fsys.Open(name) + if err != nil { + return nil, err + } + sf, ok := f.(File) // checks for io.Seeker + if !ok { + f.Close() + return nil, fmt.Errorf("open %s: fs.File must implement io.Seeker", name) + } + return sf, nil +} + +func (s *fsSource) Readlink(name string) (string, error) { + if rl, ok := s.fsys.(fs.ReadLinkFS); ok { + return rl.ReadLink(name) + } + return "", fmt.Errorf("readlink %s: fs.FS does not implement fs.ReadLinkFS", name) +} + +func (s *fsSource) Close() error { return nil } + +// subSource wraps a FileSource and serves the specified subtree, +// like fs.Sub() does for fs.FS. +type subSource struct { + underlying FileSource + subdir string + fsys fs.FS +} + +func newSubSource(underlying FileSource, subdir string) (FileSource, error) { + fsys, err := fs.Sub(underlying.FS(), subdir) + if err != nil { + return nil, err + } + return &subSource{ + underlying: underlying, + subdir: subdir, + fsys: fsys, + }, nil +} + +func (s *subSource) FS() fs.FS { return s.fsys } + +func (s *subSource) Open(name string) (File, error) { + return s.underlying.Open(path.Join(s.subdir, name)) +} + +func (s *subSource) Readlink(name string) (string, error) { + return s.underlying.Readlink(path.Join(s.subdir, name)) +} + +func (s *subSource) Close() error { return nil } diff --git a/internal/sender/stat.go b/internal/sender/stat.go new file mode 100644 index 0000000..7b70550 --- /dev/null +++ b/internal/sender/stat.go @@ -0,0 +1,17 @@ +//go:build !linux && !darwin + +package sender + +import "io/fs" + +func uidFromFileInfo(fs.FileInfo) (int32, bool) { + return 0, false +} + +func gidFromFileInfo(fs.FileInfo) (int32, bool) { + return 0, false +} + +func rdevFromFileInfo(fs.FileInfo) (int32, bool) { + return 0, false +} diff --git a/internal/sender/stat_unix.go b/internal/sender/stat_unix.go new file mode 100644 index 0000000..c16a548 --- /dev/null +++ b/internal/sender/stat_unix.go @@ -0,0 +1,32 @@ +//go:build linux || darwin + +package sender + +import ( + "io/fs" + "syscall" +) + +func uidFromFileInfo(info fs.FileInfo) (int32, bool) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return int32(st.Uid), true +} + +func gidFromFileInfo(info fs.FileInfo) (int32, bool) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return int32(st.Gid), true +} + +func rdevFromFileInfo(info fs.FileInfo) (int32, bool) { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, false + } + return int32(st.Rdev), true +} diff --git a/internal/sender/token.go b/internal/sender/token.go new file mode 100644 index 0000000..d227abd --- /dev/null +++ b/internal/sender/token.go @@ -0,0 +1,37 @@ +package sender + +// rsync/token.c:simple_send_token +func (st *Transfer) simpleSendToken(ms *mapStruct, token int32, offset int64, n int64) error { + if n > 0 { + st.Logger.Printf("sending unmatched chunks offset=%d, n=%d", offset, n) + l := int64(0) + for l < n { + n1 := min(int64(chunkSize), n-l) + + chunk, err := ms.ptr(offset+l, int32(n1)) + if err != nil { + return err + } + + if err := st.Conn.WriteInt32(int32(n1)); err != nil { + return err + } + + if _, err := st.Conn.Writer.Write(chunk); err != nil { + return err + } + + l += n1 + } + } + if token != -2 { + return st.Conn.WriteInt32(-(token + 1)) + } + return nil +} + +// rsync/token.c:send_token +func (st *Transfer) sendToken(ms *mapStruct, i int32, offset int64, n int64) error { + // TODO(compression): send deflated token + return st.simpleSendToken(ms, i, offset, n) +} diff --git a/internal/sender/transfer.go b/internal/sender/transfer.go new file mode 100644 index 0000000..ca06cc3 --- /dev/null +++ b/internal/sender/transfer.go @@ -0,0 +1,50 @@ +package sender + +import ( + "io" + + "github.com/runpod/rsync/internal/log" + "github.com/runpod/rsync/internal/progress" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncwire" +) + +type Osenv struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer +} + +// TransferOpts is a subset of Opts which is required for implementing a receiver. +type TransferOpts struct { + Verbose bool + DryRun bool + + DeleteMode bool + PreserveGid bool + PreserveUid bool + PreserveLinks bool + PreservePerms bool + PreserveDevices bool + PreserveSpecials bool + PreserveTimes bool + PreserveHardlinks bool +} + +type Transfer struct { + // config + // Opts *Opts + Logger log.Logger + Opts *rsyncopts.Options + Env *rsyncos.Env + Progress progress.Printer + Source FileSource // for modules specifying a fs.FS + + // state + Conn *rsyncwire.Conn + Seed int32 + lastMatch int64 +} + +//func (rt *Transfer) listOnly() bool { return rt.Dest == "" } diff --git a/internal/testlogger/testlogger.go b/internal/testlogger/testlogger.go new file mode 100644 index 0000000..643e49d --- /dev/null +++ b/internal/testlogger/testlogger.go @@ -0,0 +1,54 @@ +// Package testlogger contains a helper to put a stdout/stderr output stream of +// a subprocess onto the testing package's t.Log(). +package testlogger + +import ( + "bufio" + "io" + "sync" + "testing" +) + +type Logger struct { + tb testing.TB + writer *io.PipeWriter + scanner *bufio.Scanner +} + +func New(tb testing.TB) *Logger { + r, w := io.Pipe() + tl := &Logger{ + tb: tb, + writer: w, + scanner: bufio.NewScanner(r), + } + var wg sync.WaitGroup + wg.Add(1) + tb.Cleanup(func() { + w.Close() + // tl.scanner.Scan() will return false, + // tl.scanner.Err() will return nil. + + // Ensure the goroutine below is done + // to prevent data races in tb.Log() + wg.Wait() + }) + go func() { + defer wg.Done() + for tl.scanner.Scan() { + tb.Log(tl.scanner.Text()) + } + if err := tl.scanner.Err(); err != nil { + tb.Log(err) + } + }() + return tl +} + +// Write implements io.Writer. +func (lw Logger) Write(p []byte) (n int, err error) { + return lw.writer.Write(p) +} + +// Close implements io.Closer. +func (lw Logger) Close() error { return nil } diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..59e372d --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,14 @@ +package version + +import ( + "runtime/debug" +) + +func Read() string { + info, ok := debug.ReadBuildInfo() + mainVersion := info.Main.Version + if !ok { + mainVersion = "" + } + return "gokrazy/rsync " + mainVersion +} diff --git a/logger.go b/logger.go new file mode 100644 index 0000000..8fe4f98 --- /dev/null +++ b/logger.go @@ -0,0 +1,7 @@ +package rsync + +import "github.com/runpod/rsync/internal/log" + +// Logger is an interface that allows specifying your own logger. +// By default, the Go log package is used, which prints to stderr. +type Logger = log.Logger diff --git a/rsyncclient/rsyncclient.go b/rsyncclient/rsyncclient.go new file mode 100644 index 0000000..8df4088 --- /dev/null +++ b/rsyncclient/rsyncclient.go @@ -0,0 +1,149 @@ +// Package rsyncclient implements an rsync client (only), but note that +// gokrazy/rsync contains a native Go rsync implementation that supports sending +// and receiving files as client or server, compatible with the original tridge +// rsync (from the samba project) or openrsync (used on OpenBSD and macOS 15+). +package rsyncclient + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/runpod/rsync/internal/maincmd" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncstats" +) + +// Option specifies the client options. +type Option interface { + applyServer(*Client) +} + +type clientOptionFunc func(server *Client) + +func (f clientOptionFunc) applyServer(s *Client) { + f(s) +} + +// WithStderr makes the [Client] write to the specified stderr instead of +// [os.Stderr]. +func WithStderr(stderr io.WriteCloser) Option { + return clientOptionFunc(func(c *Client) { + c.osenv.Stderr = stderr + }) +} + +// WithSender enables sender mode (receiver by default). +func WithSender() Option { + return clientOptionFunc(func(c *Client) { + c.sender = true + }) +} + +// WithoutNegotiate disables protocol version negotiation (enabled by default). +func WithoutNegotiate() Option { + return clientOptionFunc(func(c *Client) { + c.negotiate = false + }) +} + +func DontRestrict() Option { + return clientOptionFunc(func(c *Client) { + c.osenv.DontRestrict = true + }) +} + +type Client struct { + osenv *rsyncos.Env + opts *rsyncopts.Options + negotiate bool + sender bool +} + +// New creates a new [Client]. You can call [Client.Run] one or more times with +// the same [Client]. +func New(args []string, opts ...Option) (*Client, error) { + c := &Client{ + osenv: &rsyncos.Env{ + Stdout: os.Stdout, + Stderr: os.Stderr, + }, + negotiate: true, + } + + for _, opt := range opts { + opt.applyServer(c) + } + + pc := rsyncopts.NewContext(rsyncopts.NewOptionsWithGokrazyDefaults(c.osenv)) + if err := pc.ParseArguments(c.osenv, args); err != nil { + return nil, err + } + c.opts = pc.Options + if len(pc.RemainingArgs) > 0 { + return nil, fmt.Errorf("remaining args %q not permitted; specify them in Client.Run()", pc.RemainingArgs) + } + if c.sender { + c.opts.SetSender() + } + + return c, nil +} + +// ServerCommandOptions returns the options that rsync would use to spawn the +// server process. +func (c *Client) ServerCommandOptions(path string, paths ...string) []string { + return c.opts.CommandOptions(path, paths...) +} + +// Result contains information about a transfer. +type Result struct { + Stats *rsyncstats.TransferStats +} + +// Run starts one run of the rsync protocol (not the rsync daemon protocol), see +// also https://michael.stapelberg.ch/posts/2022-07-02-rsync-how-does-it-work/. +// +// If you just want to transfer some data from an already running rsync server +// or remote system, use the [github.com/runpod/rsync/rsynccmd] package +// instead, which will take care of starting rsync as a subprocess (locally or +// remotely), or of connecting to an rsync daemon via TCP. +// +// The Run method operates on any kind of connection (using the +// [io.ReadWriteCloser] interface) and is meant to be used when you need more +// control over the setup. For example, maybe you want to set up some custom +// tunneling to an rsync process running deep in some remote cloud +// infrastructure. +// +// Or maybe you want to connect an rsync client and server to each other via a +// custom RPC protocol. In that case, you will need to transport the +// [Client.ServerCommandOptions] to the server and then arrange for two +// [io.ReadWriter] connections between client and server. +func (c *Client) Run(ctx context.Context, conn io.ReadWriteCloser, paths []string) (*Result, error) { + stats, err := maincmd.ClientRun(c.osenv, c.opts, conn, paths, c.negotiate) + if err != nil { + return nil, err + } + return &Result{Stats: stats}, nil +} + +// RunDaemon starts one run of the rsync daemon protocol, meaning it performs +// the daemon protocol inband exchange (to negotiate the protocol version and +// select an rsync module) and then calls [Client.Run]. +// +// This method is useful when you want to connect to an rsync daemon, but +// establish the connection yourself, e.g. via the [golang.org/x/crypto/ssh] +// package. +func (c *Client) RunDaemon(ctx context.Context, conn io.ReadWriteCloser, remotePath string, paths []string) (*Result, error) { + done, err := maincmd.StartInbandExchange(c.osenv, c.opts, conn, remotePath) + if err != nil { + return nil, err + } + if done { // Server sent EXIT + return &Result{Stats: &rsyncstats.TransferStats{}}, nil + } + c.negotiate = false // done as part of the inband exchange + return c.Run(ctx, conn, paths) +} diff --git a/rsyncclient/rsyncclient_test.go b/rsyncclient/rsyncclient_test.go new file mode 100644 index 0000000..88ca69e --- /dev/null +++ b/rsyncclient/rsyncclient_test.go @@ -0,0 +1,337 @@ +package rsyncclient_test + +import ( + "bytes" + "context" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncostest" + "github.com/runpod/rsync/internal/rsynctest" + "github.com/runpod/rsync/internal/testlogger" + "github.com/runpod/rsync/rsyncclient" + "github.com/runpod/rsync/rsyncd" +) + +func ExampleClient_Run_receiveFromSubprocess() { + args, src, dest := []string{"-av"}, "/usr/share/man", "/tmp/man" + client, err := rsyncclient.New(args) + if err != nil { + log.Fatal(err) + } + + // Start an rsynccmd server and run an rsynccmd client on its stdin/stdout. + rsynccmd := exec.Command("rsync", client.ServerCommandOptions(src)...) + stdin, err := rsynccmd.StdinPipe() + if err != nil { + log.Fatal(err) + } + stdout, err := rsynccmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + if err := rsynccmd.Start(); err != nil { + log.Fatal(err) + } + // Create an io.ReadWriteCloser from a ReadCloser and a WriteCloser. + rw := &rsync.BothCloser{ + ReadCloser: stdout, // The client reads from the server's stdout. + WriteCloser: stdin, // The client writes to the server's stdin. + } + if _, err := client.Run(context.Background(), rw, []string{dest}); err != nil { + log.Fatal(err) + } +} + +func ExampleClient_Run_sendToGoroutine() { + ctx := context.Background() + + args, src, dest := []string{"-av"}, "/usr/share/man", "/tmp/man" + client, err := rsyncclient.New(args, rsyncclient.WithSender()) + if err != nil { + log.Fatal(err) + } + + // Start an srv server and run an srv client on + // an io.Pipe()-backend stdin/stdout. + srv, err := rsyncd.NewServer(nil) + if err != nil { + log.Fatal(err) + } + // stdin from the view of the rsync server + stdinrd, stdinwr := io.Pipe() + stdoutrd, stdoutwr := io.Pipe() + go func() { + conn := rsyncd.NewConnection(stdinrd, stdoutwr, "") + osenv := &rsyncos.Env{Stderr: os.Stderr} + pc := rsyncopts.NewContext(rsyncopts.NewOptions(osenv)) + if err := pc.ParseArguments(osenv, client.ServerCommandOptions(dest)); err != nil { + log.Fatalf("parsing server args: %v", err) + } + if err := srv.InternalHandleConn(ctx, conn, nil, pc); err != nil { + log.Fatal(err) + } + }() + + // Create an io.ReadWriteCloser from a ReadCloser and a WriteCloser. + rw := &rsync.BothCloser{ + ReadCloser: stdoutrd, // The client reads from the server's stdout. + WriteCloser: stdinwr, // The client writes to the server's stdin. + } + + if _, err := client.Run(ctx, rw, []string{src}); err != nil { + log.Fatal(err) + } +} + +func TestClientCommand(t *testing.T) { + t.Parallel() + + client, err := rsyncclient.New([]string{"-av"}) + if err != nil { + t.Fatal(err) + } + + tmp := t.TempDir() + dest := filepath.Join(tmp, "dest") + // Start an rsynccmd process directly for the server part of the test. + rsynccmd := exec.Command(rsynctest.AnyRsync(t), client.ServerCommandOptions(".")...) + rsynccmd.Dir = tmp + wc, err := rsynccmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + rc, err := rsynccmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + // Create an io.ReadWriteCloser from a ReadCloser and a WriteCloser. + rw := &rsync.BothCloser{ + ReadCloser: rc, + WriteCloser: wc, + } + if err := rsynccmd.Start(); err != nil { + t.Fatal(err) + } + + if _, err := client.Run(t.Context(), rw, []string{dest}); err != nil { + t.Fatal(err) + } +} + +func TestClientServerModule(t *testing.T) { + t.Parallel() + + stderr := testlogger.New(t) + tmp := t.TempDir() + + src := filepath.Join(tmp, "src") + dest := filepath.Join(tmp, "dest") + const hello = "world" + if err := os.MkdirAll(src, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "hello"), []byte(hello), 0644); err != nil { + t.Fatal(err) + } + + args := []string{"-av"} + client, err := rsyncclient.New(args, rsyncclient.WithStderr(stderr)) + if err != nil { + t.Fatal(err) + } + + mod := rsyncd.Module{ + Name: "tmp", + Path: src, + } + srv, err := rsyncd.NewServer([]rsyncd.Module{mod}, rsyncd.WithStderr(stderr)) + if err != nil { + t.Fatal(err) + } + // stdin from the view of the rsync server + stdinrd, stdinwr := io.Pipe() + stdoutrd, stdoutwr := io.Pipe() + conn := rsyncd.NewConnection(stdinrd, stdoutwr, "") + osenv := rsyncostest.New(t) + pc := rsyncopts.NewContext(rsyncopts.NewOptions(osenv)) + if err := pc.ParseArguments(osenv, client.ServerCommandOptions("./")); err != nil { + t.Fatalf("parsing server args: %v", err) + } + t.Logf("pc.RemainingArgs=%q", pc.RemainingArgs) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := srv.InternalHandleConn(t.Context(), conn, &mod, pc) + if err != nil { + t.Error(err) + } + }() + + // Create an io.ReadWriteCloser from a ReadCloser and a WriteCloser. + rw := &rsync.BothCloser{ + ReadCloser: stdoutrd, + WriteCloser: stdinwr, + } + if _, err := client.Run(t.Context(), rw, []string{dest}); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(hello)) { + t.Errorf("hello: unexpected contents: diff (-want +got):\n%s", cmp.Diff([]byte(hello), got)) + } + + // Ensure an error would be displayed, if any. + wg.Wait() +} + +// like TestClientServerModule, but without a module, +// i.e. using the command calling convention. +func TestClientServerCommand(t *testing.T) { + t.Parallel() + + stderr := testlogger.New(t) + tmp := t.TempDir() + + src := filepath.Join(tmp, "src") + "/" + dest := filepath.Join(tmp, "dest") + const hello = "world" + if err := os.MkdirAll(src, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "hello"), []byte(hello), 0644); err != nil { + t.Fatal(err) + } + + args := []string{"-av"} + client, err := rsyncclient.New(args) + if err != nil { + t.Fatal(err) + } + + srv, err := rsyncd.NewServer(nil, rsyncd.WithStderr(stderr)) + if err != nil { + t.Fatal(err) + } + // stdin from the view of the rsync server + stdinrd, stdinwr := io.Pipe() + stdoutrd, stdoutwr := io.Pipe() + conn := rsyncd.NewConnection(stdinrd, stdoutwr, "") + osenv := rsyncostest.New(t) + pc := rsyncopts.NewContext(rsyncopts.NewOptions(osenv)) + if err := pc.ParseArguments(osenv, client.ServerCommandOptions(src)); err != nil { + t.Fatalf("parsing server args: %v", err) + } + t.Logf("pc.RemainingArgs=%q", pc.RemainingArgs) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := srv.InternalHandleConn(t.Context(), conn, nil, pc) + if err != nil { + t.Error(err) + } + }() + + // Create an io.ReadWriteCloser from a ReadCloser and a WriteCloser. + rw := &rsync.BothCloser{ + ReadCloser: stdoutrd, + WriteCloser: stdinwr, + } + if _, err := client.Run(t.Context(), rw, []string{dest}); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(hello)) { + t.Errorf("hello: unexpected contents: diff (-want +got):\n%s", cmp.Diff([]byte(hello), got)) + } + + // Ensure an error would be displayed, if any. + wg.Wait() +} + +// like TestClientServerCommand, but sending data instead of receiving. +func TestClientServerCommandSender(t *testing.T) { + t.Parallel() + + stderr := testlogger.New(t) + tmp := t.TempDir() + + src := filepath.Join(tmp, "src") + "/" + dest := filepath.Join(tmp, "dest") + const hello = "world" + if err := os.MkdirAll(src, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "hello"), []byte(hello), 0644); err != nil { + t.Fatal(err) + } + + args := []string{"-av"} + client, err := rsyncclient.New(args, rsyncclient.WithSender()) + if err != nil { + t.Fatal(err) + } + + srv, err := rsyncd.NewServer(nil, rsyncd.WithStderr(stderr)) + if err != nil { + t.Fatal(err) + } + // stdin from the view of the rsync server + stdinrd, stdinwr := io.Pipe() + stdoutrd, stdoutwr := io.Pipe() + conn := rsyncd.NewConnection(stdinrd, stdoutwr, "") + osenv := rsyncostest.New(t) + pc := rsyncopts.NewContext(rsyncopts.NewOptions(osenv)) + if err := pc.ParseArguments(osenv, client.ServerCommandOptions(dest)); err != nil { + t.Fatalf("parsing server args: %v", err) + } + t.Logf("pc.RemainingArgs=%q", pc.RemainingArgs) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := srv.InternalHandleConn(t.Context(), conn, nil, pc) + if err != nil { + t.Error(err) + } + }() + + // Create an io.ReadWriteCloser from a ReadCloser and a WriteCloser. + rw := &rsync.BothCloser{ + ReadCloser: stdoutrd, + WriteCloser: stdinwr, + } + if _, err := client.Run(t.Context(), rw, []string{src}); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(filepath.Join(dest, "hello")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, []byte(hello)) { + t.Errorf("hello: unexpected contents: diff (-want +got):\n%s", cmp.Diff([]byte(hello), got)) + } + + // Ensure an error would be displayed, if any. + wg.Wait() +} diff --git a/rsynccmd/rsynccmd.go b/rsynccmd/rsynccmd.go new file mode 100644 index 0000000..3200c51 --- /dev/null +++ b/rsynccmd/rsynccmd.go @@ -0,0 +1,71 @@ +// Package rsynccmd provides a command-like interface to gokrazy/rsync, which +// contains a native Go rsync implementation that supports sending and receiving +// files as client or server, compatible with the original tridge rsync (from +// the samba project) or openrsync (used on OpenBSD and macOS 15+). +// +// This interface allows you to replace calls to an external rsync program, like: +// +// rsync -a rsync://share01/dataset /tmp/dataset +// +// …with calls into Go code running in the same process (no dependency on an +// external rsync program!): +// +// cmd := rsynccmd.Command("rsync", "-a", "rsync://share01/dataset", "/tmp/dataset") +// cmd.Stdout = os.Stdout +// cmd.Stderr = os.Stderr +// if _, err := cmd.Run(context.Background()); err != nil { +// return fmt.Errorf("%v: %v", cmd.Args, err) +// } +package rsynccmd + +import ( + "context" + "io" + "net" + + "github.com/runpod/rsync/internal/maincmd" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncstats" +) + +// Cmd represents an rsync invocation being prepared or run. +type Cmd struct { + Args []string + Stdin io.ReadCloser + Stdout io.WriteCloser + Stderr io.WriteCloser + DontRestrict bool + + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) +} + +// Command returns the [Cmd] struct to execute rsync with the given arguments. +// +// The name parameter is ignored and only specified for symmetry with +// os/exec.Command. +func Command(name string, arg ...string) *Cmd { + return &Cmd{ + Args: append([]string{name}, arg...), + } +} + +// Result contains information about the transfer. +type Result struct { + Stats *rsyncstats.TransferStats +} + +// Run starts the specified rsync invocation. +func (c *Cmd) Run(ctx context.Context) (*Result, error) { + osenv := &rsyncos.Env{ + Stdin: c.Stdin, + Stdout: c.Stdout, + Stderr: c.Stderr, + DontRestrict: c.DontRestrict, + DialContext: c.DialContext, + } + stats, err := maincmd.Main(ctx, osenv, c.Args, nil) + if err != nil { + return nil, err + } + return &Result{Stats: stats}, nil +} diff --git a/rsyncd/restrictmodules.go b/rsyncd/restrictmodules.go new file mode 100644 index 0000000..206db29 --- /dev/null +++ b/rsyncd/restrictmodules.go @@ -0,0 +1,26 @@ +package rsyncd + +import ( + "fmt" + "os" + + "github.com/runpod/rsync/internal/restrict" +) + +func restrictToModules(modules []Module) error { + var roDirs, rwDirs []string + for _, mod := range modules { + if mod.FS != nil { + continue + } + if mod.Writable { + if err := os.MkdirAll(mod.Path, 0755); err != nil { + return fmt.Errorf("MkdirAll(mod=%s): %v", mod.Name, err) + } + rwDirs = append(rwDirs, mod.Path) + } else { + roDirs = append(roDirs, mod.Path) + } + } + return restrict.MaybeFileSystem(roDirs, rwDirs) +} diff --git a/rsyncd/rsyncd.go b/rsyncd/rsyncd.go new file mode 100644 index 0000000..a8e16ff --- /dev/null +++ b/rsyncd/rsyncd.go @@ -0,0 +1,641 @@ +// Package rsyncd implements an rsync server (only), but note that gokrazy/rsync +// contains a native Go rsync implementation that supports sending and receiving +// files as client or server, compatible with the original tridge rsync (from +// the samba project) or openrsync (used on OpenBSD and macOS 15+). +package rsyncd + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "io/fs" + "net" + "os" + "path/filepath" + "strings" + "time" + + "github.com/runpod/rsync" + "github.com/runpod/rsync/internal/log" + "github.com/runpod/rsync/internal/progress" + "github.com/runpod/rsync/internal/receiver" + "github.com/runpod/rsync/internal/rsyncopts" + "github.com/runpod/rsync/internal/rsyncos" + "github.com/runpod/rsync/internal/rsyncwire" + "github.com/runpod/rsync/internal/sender" +) + +type Module struct { + Name string `toml:"name"` + Path string `toml:"path"` // If empty, FS must be non-nil + FS fs.FS `toml:"-"` // If set, serve from this instead of Path + ACL []string `toml:"acl"` + Writable bool `toml:"writable"` // Must be false if FS is set +} + +// Option specifies the server options. +type Option interface { + applyServer(*Server) +} + +type serverOptionFunc func(server *Server) + +func (f serverOptionFunc) applyServer(s *Server) { + f(s) +} + +// WithLogger specifies the logger to use for the server. +// It also sets the global logger used by the rsync package. +func WithLogger(logger log.Logger) Option { + return serverOptionFunc(func(s *Server) { + s.logger = logger + }) +} + +func WithStderr(stderr io.WriteCloser) Option { + return serverOptionFunc(func(s *Server) { + s.stderr = stderr + }) +} + +func DontRestrict() Option { + return serverOptionFunc(func(s *Server) { + s.dontRestrict = true + }) +} + +func NewServer(modules []Module, opts ...Option) (*Server, error) { + for _, mod := range modules { + if err := validateModule(mod); err != nil { + return nil, err + } + } + + server := &Server{ + modules: modules, + } + + for _, opt := range opts { + opt.applyServer(server) + } + + // Default to os.Stderr if no stderr was specified. + // Explicitly use io.Discard if you do not want stderr. + if server.stderr == nil { + server.stderr = os.Stderr + } + + if server.logger == nil { + // TODO: use the logger in a *rsyncos.Env instead + server.logger = log.New(server.stderr) + } + + // An empty module list means this server is a sender + // (e.g. started in command mode with --server --sender), + // in which case restrict.MaybeFileSystem() will be called + // by the caller of NewServer(). + if !server.dontRestrict && len(server.modules) > 0 { + if err := restrictToModules(server.modules); err != nil { + return nil, err + } + } + + return server, nil +} + +type Server struct { + stderr io.WriteCloser + logger log.Logger + dontRestrict bool + + modules []Module +} + +func (s *Server) getModule(requestedModule string) (Module, error) { + for _, mod := range s.modules { + if mod.Name == requestedModule { + return mod, nil + } + } + + return Module{}, fmt.Errorf("no such module: %s", requestedModule) +} + +func (s *Server) formatModuleList() string { + if len(s.modules) == 0 { + return "" + } + var list strings.Builder + for _, mod := range s.modules { + comment := mod.Name // for now + fmt.Fprintf(&list, "%s\t%s\n", + mod.Name, + comment) + } + return list.String() +} + +func checkACL(acls []string, remoteAddr string) error { + if len(acls) == 0 { + return nil + } + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return fmt.Errorf("BUG: invalid remote address %q", remoteAddr) + } + remoteIP := net.ParseIP(host) + if remoteIP == nil { + return fmt.Errorf("BUG: invalid remote host %q", host) + } + for _, acl := range acls { + // TODO(performance): move ACL parsing to config-time to make ACL checks + // less expensive + i := strings.Index(acl, " ") + if i < 0 { + return fmt.Errorf("invalid acl: %q (no space found)", acl) + } + action, who := acl[:i], acl[i+len(" "):] + if action != "allow" && action != "deny" { + return fmt.Errorf("invalid acl: %q (syntax: allow|deny )", acl) + } + if who == "all" { + // The all keyword matches any remote IP address + } else { + _, net, err := net.ParseCIDR(who) + if err != nil { + return fmt.Errorf("invalid acl: %q (syntax: allow|deny )", acl) + } + if !net.Contains(remoteIP) { + // Skip this instruction, the remote IP does not match + continue + } + } + switch action { + case "allow": + return nil + case "deny": + return fmt.Errorf("access denied (acl %q)", acl) + default: + return fmt.Errorf("invalid acl: %q (syntax: allow|deny )", acl) + } + } + return nil +} + +// FIXME: context cancellation not yet implemented +func (s *Server) HandleDaemonConn(ctx context.Context, conn *Conn) (err error) { + _ = ctx // not implemented. what would be the best thing to do? wrap conn's reader part with cancelable reader? + + const terminationCommand = "@RSYNCD: OK\n" + cwr := conn.cwr + rd := conn.rd + // send server greeting + + fmt.Fprintf(cwr, "@RSYNCD: %d\n", rsync.ProtocolVersion) + + // read client greeting + clientGreeting, err := rd.ReadString('\n') + if err != nil { + return err + } + if !strings.HasPrefix(clientGreeting, "@RSYNCD: ") { + return fmt.Errorf("invalid client greeting: got %q", clientGreeting) + } + // TODO: protocol negotiation + + // read requested module(s), if any + requestedModule, err := rd.ReadString('\n') + if err != nil { + return err + } + requestedModule = strings.TrimSpace(requestedModule) + if requestedModule == "" || requestedModule == "#list" { + s.logger.Printf("client %v requested rsync module listing", conn.name) + io.WriteString(cwr, s.formatModuleList()) + io.WriteString(cwr, "@RSYNCD: EXIT\n") + return nil + } + s.logger.Printf("client %v requested rsync module %q", conn.name, requestedModule) + module, err := s.getModule(requestedModule) + if err != nil { + fmt.Fprintf(cwr, "@ERROR: Unknown module %q\n", requestedModule) + return err + } + + if err := checkACL(module.ACL, conn.name); err != nil { + fmt.Fprintf(cwr, "@ERROR: %v\n", err) + return err + } + + io.WriteString(cwr, terminationCommand) + + // read requested flags + var flags []string + for { + flag, err := rd.ReadString('\n') + if err != nil { + return err + } + flag = strings.TrimSpace(flag) + s.logger.Printf("client sent: %q", flag) + if flag == "" { + break + } + flags = append(flags, flag) + } + + s.logger.Printf("flags: %+v", flags) + osenv := &rsyncos.Env{Stderr: s.stderr} + pc := rsyncopts.NewContext(rsyncopts.NewOptionsWithGokrazyDefaults(osenv)) + if err := pc.ParseArguments(osenv, flags); err != nil { + err = fmt.Errorf("parsing server args: %v", err) + + // terminate connection with an error about which flag is not supported + c := &rsyncwire.Conn{ + Reader: io.NopCloser(rd), + Writer: cwr, + } + + const errorSeed = 0xee + if err := c.WriteInt32(errorSeed); err != nil { + return err + } + + // Switch to multiplexing protocol, but only for server-side transmissions. + // Transmissions received from the client are not multiplexed. + mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer} + mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [sender]: %v\n", err)) + + return err + } + remaining := pc.RemainingArgs + s.logger.Printf("remaining: %q", remaining) + // remaining[0] is always "." + // remaining[1] is the first directory + if len(remaining) < 2 { + return fmt.Errorf("invalid args: at least one directory required") + } + if got, want := remaining[0], "."; got != want { + return fmt.Errorf("protocol error: got %q, expected %q", got, want) + } + paths := remaining[1:] + s.logger.Printf("paths: %q", paths) + + // Strip the module_name/ prefix out of the paths, + // see rsync/io.c:read_args, glob_expand_module(). + for idx, path := range pc.RemainingArgs { + if idx == 0 { + // skip pc.RemainingArgs[0], only strip RemainingArgs[1:] + continue + } + trimmed := strings.TrimPrefix(path, module.Name) + if trimmed == "" { + trimmed = "." + } + pc.RemainingArgs[idx] = trimmed + } + + s.logger.Printf("trimmed paths: %q", pc.RemainingArgs[1:]) + + return s.handleConn(ctx, conn, &module, pc, false) +} + +type Conn struct { + name string + crd *rsyncwire.CountingReader + cwr *rsyncwire.CountingWriter + rd *bufio.Reader + closer io.Closer +} + +func NewConnection(r io.ReadCloser, w io.WriteCloser, name string) *Conn { + crd, cwr := rsyncwire.CounterPair(r, w) + rd := bufio.NewReader(crd) + return &Conn{ + name: name, + crd: crd, + cwr: cwr, + rd: rd, + closer: &rsync.BothCloser{ + ReadCloser: r, + WriteCloser: w, + }, + } +} + +// This method is only exported until we refactor; use HandleConnArgs() instead +func (s *Server) InternalHandleConn(ctx context.Context, conn *Conn, module *Module, pc *rsyncopts.Context) error { + return s.handleConn(ctx, conn, module, pc, true /* negotiate */) +} + +func (s *Server) HandleConnArgs(ctx context.Context, conn *Conn, module *Module, args []string) error { + osenv := &rsyncos.Env{Stderr: s.stderr} + pc := rsyncopts.NewContext(rsyncopts.NewOptionsWithGokrazyDefaults(osenv)) + if err := pc.ParseArguments(osenv, args); err != nil { + return fmt.Errorf("parsing server args: %v", err) + } + return s.handleConn(ctx, conn, module, pc, true /* negotiate */) +} + +// handleConn is equivalent to rsync/main.c:start_server +func (s *Server) handleConn(ctx context.Context, conn *Conn, module *Module, pc *rsyncopts.Context, negotiate bool) (err error) { + rd := conn.rd + crd := conn.crd + cwr := conn.cwr + opts := pc.Options + paths := pc.RemainingArgs[1:] + + // “SHOULD be unique to each connection” as per + // https://github.com/JohannesBuchner/Jarsync/blob/master/jarsync/rsync.txt + // + // Computed the same way that tridge rsync does it, but the details do not + // matter. The goal is to have a checksum seed each time. + sessionChecksumSeed := int32(time.Now().Unix()) ^ (int32(os.Getpid()) << 6) + + c := &rsyncwire.Conn{ + Reader: struct { + io.Reader + io.Closer + }{ + Reader: rd, + Closer: conn.closer, + }, + Writer: cwr, + } + + if negotiate { + remoteProtocol, err := c.ReadInt32() + if err != nil { + return err + } + if opts.DebugGTE(rsyncopts.DEBUG_PROTO, 1) { + s.logger.Printf("remote protocol: %d", remoteProtocol) + } + if err := c.WriteInt32(rsync.ProtocolVersion); err != nil { + return err + } + } + + if err := c.WriteInt32(sessionChecksumSeed); err != nil { + return err + } + + // Switch to multiplexing protocol, but only for server-side transmissions. + // Transmissions received from the client are not multiplexed. + mpx := &rsyncwire.MultiplexWriter{Writer: c.Writer} + // Update cwr to track the multiplexed writer, + // but copy the number of bytes written. + cwr = &rsyncwire.CountingWriter{ + W: mpx, + BytesWritten: cwr.BytesWritten, + } + c.Writer = cwr + + if opts.Sender() { + // If returning an error, send the error to the client for display, too: + defer func() { + if err != nil { + mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [sender]: %v\n", err)) + } + }() + + return s.handleConnSender(module, crd, cwr, paths, opts, false, c, sessionChecksumSeed) + } + + // If returning an error, send the error to the client for display, too: + defer func() { + if err != nil { + mpx.WriteMsg(rsyncwire.MsgError, fmt.Appendf(nil, "gokr-rsync [receiver]: %v\n", err)) + } + }() + return s.handleConnReceiver(module, crd, cwr, paths, opts, false, c, sessionChecksumSeed) +} + +// handleConnReceiver is equivalent to rsync/main.c:do_server_recv +func (s *Server) handleConnReceiver(module *Module, crd *rsyncwire.CountingReader, cwr *rsyncwire.CountingWriter, paths []string, opts *rsyncopts.Options, negotiate bool, c *rsyncwire.Conn, sessionChecksumSeed int32) (err error) { + var destPath string + implicitModule := module == nil + if implicitModule { + if len(paths) != 1 { + return fmt.Errorf("precisely one destination path required, got %q", paths) + } + module = &Module{ + Name: "implicit", + Path: paths[0], + Writable: true, + } + destPath = module.Path + } + if opts.Verbose() { + s.logger.Printf("handleConnReceiver(module=%+v, destPath=%q)", module, destPath) + } + + if !module.Writable { + return fmt.Errorf("ERROR: module is read only") + } + + rt := &receiver.Transfer{ + Logger: s.logger, + Opts: &receiver.TransferOpts{ + DryRun: opts.DryRun(), + Server: opts.Server(), + Verbose: opts.Verbose(), + Progress: opts.Progress(), + + DeleteMode: opts.DeleteMode(), + PreserveGid: opts.PreserveGid(), + PreserveUid: opts.PreserveUid(), + PreserveLinks: opts.PreserveLinks(), + PreservePerms: opts.PreservePerms(), + PreserveDevices: opts.PreserveDevices(), + PreserveSpecials: opts.PreserveSpecials(), + PreserveTimes: opts.PreserveMTimes(), + // TODO: PreserveHardlinks: opts.PreserveHardlinks, + IgnoreTimes: opts.IgnoreTimes(), + AlwaysChecksum: opts.AlwaysChecksum(), + DoFsync: opts.DoFsync(), + + InfoGTE: opts.InfoGTE, + DebugGTE: opts.DebugGTE, + + KeepPartial: opts.KeepPartial(), + }, + Dest: module.Path, + Env: &rsyncos.Env{ + Stderr: s.stderr, + }, + Conn: c, + Seed: sessionChecksumSeed, + Progress: progress.NewPrinter(io.Discard, time.Now), + } + if err := os.MkdirAll(rt.Dest, 0755); err != nil { + return fmt.Errorf("MkdirAll(dest=%s): %v", rt.Dest, err) + } + rt.DestRoot, err = os.OpenRoot(rt.Dest) + if err != nil { + return fmt.Errorf("OpenRoot(dest=%s): %v", rt.Dest, err) + } + defer rt.DestRoot.Close() + + if !implicitModule { + if len(paths) > 1 { + return fmt.Errorf("module is available, and at most one destination path is allowed, got %q", paths) + } + // Descend into subdirectory (if requested), + // using the os.OpenRoot traversal-safe API. + if len(paths) == 1 && paths[0] != "/" { + // Trim leading and trailing slash: + // (*os.Root).MkdirAll("foo/") was a security issue: + // https://go.dev/issue/79005, so no longer works. + subdir := strings.Trim(paths[0], "/") + subRoot, err := rt.DestRoot.OpenRoot(subdir) + if err != nil { + if os.IsNotExist(err) { + if err := rt.DestRoot.MkdirAll(subdir, 0755); err != nil { + return fmt.Errorf("MkdirAll(%s): %v", subdir, err) + } + subRoot, err = rt.DestRoot.OpenRoot(subdir) + } + if err != nil { + return fmt.Errorf("OpenRoot(%s): %v", subdir, err) + } + } + if name := subRoot.Name(); filepath.IsAbs(name) { + rt.Dest = name + } else { + // Go changed behavior: In Go 1.25, subRoot.Name() + // did not return an absolute path: + // https://go.googlesource.com/go/+/ed7f804 + rt.Dest = filepath.Join(rt.Dest, name) + } + rt.DestRoot = subRoot + if opts.Verbose() { + s.logger.Printf("opened subdirectory %q", rt.Dest) + } + } + } + + if opts.PreserveHardLinks() { + return fmt.Errorf("support for hard links not yet implemented") + } + + if opts.DeleteMode() { + // receive the exclusion list (openrsync’s is always empty) + exclusionList, err := sender.RecvFilterList(c) + if err != nil { + return err + } + s.logger.Printf("exclusion list read (entries: %d)", len(exclusionList.Filters)) + } + + // receive file list + if opts.InfoGTE(rsyncopts.INFO_FLIST, 1) { + s.logger.Printf("receiving file list") + } + fileList, err := rt.ReceiveFileList() + if err != nil { + return err + } + if opts.InfoGTE(rsyncopts.INFO_FLIST, 1) { + s.logger.Printf("received %d names", len(fileList)) + } + stats, err := rt.Do(c, fileList, true) + if err != nil { + return err + } + if opts.InfoGTE(rsyncopts.INFO_STATS, 1) { + s.logger.Printf("stats: %+v", stats) + } + return nil +} + +// handleConnSender is equivalent to rsync/main.c:do_server_sender +func (s *Server) handleConnSender(module *Module, crd *rsyncwire.CountingReader, cwr *rsyncwire.CountingWriter, paths []string, opts *rsyncopts.Options, negotiate bool, c *rsyncwire.Conn, sessionChecksumSeed int32) (err error) { + if module == nil { + module = &Module{ + Name: "implicit", + Path: rsync.FileSystemRoot, + } + } + + st := &sender.Transfer{ + Logger: s.logger, + Opts: opts, + Conn: c, + Seed: sessionChecksumSeed, + Env: &rsyncos.Env{ + Stderr: s.stderr, + }, + Progress: progress.NewPrinter(io.Discard, time.Now), + } + // receive the exclusion list (openrsync’s is always empty) + + if module.FS != nil { + st.Source = sender.NewFSSource(module.FS) + } + + exclusionList, err := sender.RecvFilterList(st.Conn) + if err != nil { + return err + } + st.Logger.Printf("exclusion list read (entries: %d)", len(exclusionList.Filters)) + + stats, err := st.Do(crd, cwr, module.Path, paths, exclusionList) + if err != nil { + return err + } + + s.logger.Printf("handleConnSender done. stats: %+v", stats) + + return nil +} + +func (s *Server) Serve(ctx context.Context, ln net.Listener) error { + go func() { + <-ctx.Done() + ln.Close() // unblocks Accept() + }() + + for { + conn, err := ln.Accept() + if err != nil { + select { + case <-ctx.Done(): + return nil // ignore expected 'use of closed network connection' error on context cancel + default: + return err + } + } + remoteAddr := conn.RemoteAddr() + s.logger.Printf("remote connection from %s", remoteAddr) + go func() { + defer conn.Close() + c := NewConnection(conn, conn, remoteAddr.String()) + if err := s.HandleDaemonConn(ctx, c); err != nil { + s.logger.Printf("[%s] handle: %v", remoteAddr, err) + } + }() + } +} + +func validateModule(mod Module) error { + if mod.Name == "" { + return errors.New("module has no name") + } + if mod.FS != nil { + if mod.Writable { + return fmt.Errorf("module %q: FS modules cannot be writable", mod.Name) + } + if mod.Path != "" { + return fmt.Errorf("module %q: cannot specify both Path and FS", mod.Name) + } + } else { + if mod.Path == "" { + return fmt.Errorf("module %q has empty path", mod.Name) + } + } + + return nil +} diff --git a/rsyncd/rsyncd_test.go b/rsyncd/rsyncd_test.go new file mode 100644 index 0000000..529bdb1 --- /dev/null +++ b/rsyncd/rsyncd_test.go @@ -0,0 +1,70 @@ +package rsyncd_test + +import ( + "context" + "log" + "net" + "testing/fstest" + + "github.com/runpod/rsync/rsyncd" +) + +func ExampleNewServer() { + listener, err := net.Listen("tcp", "localhost:8873") + if err != nil { + log.Fatal(err) + } + + rsyncServer, err := rsyncd.NewServer([]rsyncd.Module{ + { + Name: "music", + Path: "/home/bob/Music", + }, + }) + if err != nil { + log.Fatal(err) + } + + if err := rsyncServer.Serve(context.Background(), listener); err != nil { + log.Fatal(err) + } +} + +// ExampleNewServer_fsFS shows how to serve an rsync module backed by an [fs.FS] +// instead of a filesystem path. This allows serving files from in-memory +// filesystems, embedded files, or any other [fs.FS] implementation. +func ExampleNewServer_fsFS() { + // Create an in-memory filesystem using testing/fstest. + // Any fs.FS implementation works (embed.FS, zip archives, etc.) + memfs := fstest.MapFS{ + "hello.txt": &fstest.MapFile{ + Data: []byte("Hello from fs.FS!"), + Mode: 0o644, + }, + "readme.md": &fstest.MapFile{ + Data: []byte("# Welcome\nThis is served from memory."), + Mode: 0o644, + }, + } + + listener, err := net.Listen("tcp", "localhost:8873") + if err != nil { + log.Fatal(err) + } + + rsyncServer, err := rsyncd.NewServer([]rsyncd.Module{ + { + Name: "inmemory", + FS: memfs, + }, + }) + if err != nil { + log.Fatal(err) + } + + log.Printf("listening; now run: rsync -av rsync://%s/inmemory", listener.Addr()) + + if err := rsyncServer.Serve(context.Background(), listener); err != nil { + log.Fatal(err) + } +} diff --git a/systemd/gokr-rsyncd.service b/systemd/gokr-rsyncd.service new file mode 100644 index 0000000..3740402 --- /dev/null +++ b/systemd/gokr-rsyncd.service @@ -0,0 +1,82 @@ +[Unit] +Description=gokrazy rsync server daemon (for serving files) + +[Service] +ExecStart=/usr/bin/gokr-rsync --daemon + +# See also http://0pointer.net/blog/dynamic-users-with-systemd.html +DynamicUser=yes + +# Do not establish any new connections: +PrivateNetwork=yes + +# Remove all capabilities(7), this is a stateless web server: +CapabilityBoundingSet= + +# Ensure the service can never gain new privileges: +NoNewPrivileges=yes + +# Prohibit access to any kind of namespacing: +RestrictNamespaces=yes + +# Prohibit all address families: +# TODO(https://github.com/systemd/systemd/issues/15753): restrict to none +RestrictAddressFamilies=AF_UNIX +RestrictAddressFamilies=~AF_UNIX + +# Make home directories inaccessible: +ProtectHome=true + +# Make device nodes except for /dev/null, /dev/zero, /dev/full, +# /dev/random and /dev/urandom inaccessible: +PrivateDevices=yes + +# Make users other than root and the user for this daemon inaccessible: +PrivateUsers=yes + +# Make cgroup file system hierarchy inaccessible: +ProtectControlGroups=yes + +# Deny kernel module loading: +ProtectKernelModules=yes + +# Make kernel variables (e.g. /proc/sys) read-only: +ProtectKernelTunables=yes + +# Deny hostname changing: +ProtectHostname=yes + +# Deny realtime scheduling: +RestrictRealtime=yes + +# Deny access to other user’s information in /proc: +ProtectProc=invisible + +# Only allow access to /proc pid files, no other files: +ProcSubset=pid + +# Deny access to the kernel log ring buffer +ProtectKernelLogs=yes + +# Deny setting the hardware or system clock: +ProtectClock=yes + +# This daemon must not create any new files, but set the umask to 077 just in case. +UMask=077 + +# Filter dangerous system calls. The following is listed as safe basic choice +# in systemd.exec(5): +SystemCallArchitectures=native +SystemCallFilter=@system-service +SystemCallFilter=~@privileged +SystemCallFilter=~@resources +SystemCallErrorNumber=EPERM + +# Deny kernel execution domain changing: +LockPersonality=yes + +# Deny memory mappings that are writable and executable: +MemoryDenyWriteExecute=yes + +# no-op for a socket-activated unit, but better for systemd-analyze security: +IPAddressDeny=any diff --git a/systemd/gokr-rsyncd.socket b/systemd/gokr-rsyncd.socket new file mode 100644 index 0000000..ba85d2b --- /dev/null +++ b/systemd/gokr-rsyncd.socket @@ -0,0 +1,8 @@ +[Unit] +Description=gokrazy rsync server socket + +[Socket] +ListenStream=127.0.0.1:873 + +[Install] +WantedBy=sockets.target diff --git a/testdata/ci-debian.Dockerfile b/testdata/ci-debian.Dockerfile new file mode 100644 index 0000000..ef1eede --- /dev/null +++ b/testdata/ci-debian.Dockerfile @@ -0,0 +1,26 @@ +# vim:ft=Dockerfile +# Pinned to the current stable release rather than sid: sid ships rsync 3.5.0, +# whose hardened path traversal (RsyncProject/rsync#1050) fails change_dir with +# "Permission denied (13)" against the test tmp dirs. Stable stays on 3.4.x. +FROM debian:trixie + +RUN echo force-unsafe-io > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup +# Paper over occasional network flakiness of some mirrors. +RUN echo 'APT::Acquire::Retries "5";' > /etc/apt/apt.conf.d/80retry + +# NOTE: I tried exclusively using gce_debian_mirror.storage.googleapis.com +# instead of httpredir.debian.org, but the results (Fetched 123 MB in 36s (3357 +# kB/s)) are not any better than httpredir.debian.org (Fetched 123 MB in 34s +# (3608 kB/s)). Hence, let’s stick with httpredir.debian.org (default) for now. + +# Install rsync (for running tests). +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + rsync ssh git ca-certificates build-essential golang-go bmake zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +# Build openrsync (for running tests). +RUN cd /usr/src && \ + git clone https://github.com/kristapsdz/openrsync && \ + cd /usr/src/openrsync && \ + ./configure && bmake -j8 && bmake install diff --git a/testdata/ci-fedora.Dockerfile b/testdata/ci-fedora.Dockerfile new file mode 100644 index 0000000..694f4b6 --- /dev/null +++ b/testdata/ci-fedora.Dockerfile @@ -0,0 +1,17 @@ +# vim:ft=Dockerfile +# Pinned to a release whose rsync is still 3.4.x. "fedora" (latest) is now +# Fedora 44, which ships rsync 3.5.0, whose hardened path traversal +# (RsyncProject/rsync#1050) fails change_dir with "Permission denied (13)" +# against the test tmp dirs. Fedora 42 stays on rsync 3.4.1. +FROM fedora:42 + +# Install rsync (for running tests). +RUN dnf -y update && dnf -y install rsync openssh-clients go && dnf clean all + +# Enable toolchain management (and the module proxy, which is a requirement) so +# that Go 1.23 (from Fedora) will use Go 1.24 for gokrazy/rsync (or whichever +# version we specify as language/toolchain version in our go.mod). +RUN go env -w \ + GOPROXY=https://proxy.golang.org,direct \ + GOSUMDB=sum.golang.org \ + GOTOOLCHAIN=auto diff --git a/types.go b/types.go new file mode 100644 index 0000000..f7ad2ec --- /dev/null +++ b/types.go @@ -0,0 +1,86 @@ +package rsync + +import ( + "fmt" + + "github.com/runpod/rsync/internal/rsyncwire" +) + +// rsync/rsync.h:struct sum_buf +type SumBuf struct { + Offset int64 + Len int64 + Index int32 + Sum1 uint32 + Sum2 [16]byte +} + +// TODO: remove connection.go:sumHead in favor of this type +type SumHead struct { + // “number of blocks” (openrsync) + // “how many chunks” (rsync) + ChecksumCount int32 + + // “block length in the file” (openrsync) + // maximum (1 << 29) for older rsync, (1 << 17) for newer + BlockLength int32 + + // “long checksum length” (openrsync) + ChecksumLength int32 + + // “terminal (remainder) block length” (openrsync) + // RemainderLength is flength % BlockLength + RemainderLength int32 + + Sums []SumBuf +} + +func (sh *SumHead) ReadFrom(c *rsyncwire.Conn) error { + // TODO(protocol>=30): update maxBlockLen + const maxBlockLen = 1 << 29 // see rsync.h:OLD_MAX_BLOCK_SIZE + + var err error + sh.ChecksumCount, err = c.ReadInt32() + if err != nil { + return err + } + if sh.ChecksumCount < 0 { + return fmt.Errorf("invalid checksum count %d", sh.ChecksumCount) + } + + sh.BlockLength, err = c.ReadInt32() + if err != nil { + return err + } + if sh.BlockLength < 0 || sh.BlockLength > maxBlockLen { + return fmt.Errorf("invalid block length %d", sh.BlockLength) + } + + sh.ChecksumLength, err = c.ReadInt32() + if err != nil { + return err + } + // TODO(protocol>=27): update max sh.ChecksumLength check + if sh.ChecksumLength < 0 || sh.ChecksumLength > 16 { + return fmt.Errorf("invalid checksum length %d", sh.ChecksumLength) + } + + sh.RemainderLength, err = c.ReadInt32() + if err != nil { + return err + } + if sh.RemainderLength < 0 || sh.RemainderLength > sh.BlockLength { + return fmt.Errorf("invalid remainder length %d", sh.RemainderLength) + } + + return nil +} + +func (sh *SumHead) WriteTo(c *rsyncwire.Conn) error { + var buf rsyncwire.Buffer + buf.WriteInt32(sh.ChecksumCount) + buf.WriteInt32(sh.BlockLength) + buf.WriteInt32(sh.ChecksumLength) + buf.WriteInt32(sh.RemainderLength) + return c.WriteString(buf.String()) +}