Consolidate binaries; add 'stream' subcommand for direct-to-remote acquisition

.github/workflows/build.yml

@@ -38,16 +38,14 @@ jobs:
     - name: build
       run: eng/ci.sh
     - name: Run tests
-      run: sudo target/x86_64-unknown-linux-musl/release/avml --compress output.lime
+      run: sudo target/x86_64-unknown-linux-musl/release/avml acquire --compress output.lime
     - name: upload artifacts
       uses: actions/upload-artifact@v7.0.1
       with:
         name: linux-x86-64-artifacts
         path: |
           target/*/release/avml
           target/*/release/avml-minimal
-          target/*/release/avml-convert
-          target/*/release/avml-upload
   arm64:
     permissions:
         contents: read
@@ -58,16 +56,14 @@ jobs:
     - name: build
       run: eng/ci.sh
     - name: Run tests
-      run: sudo target/aarch64-unknown-linux-musl/release/avml --compress output.lime
+      run: sudo target/aarch64-unknown-linux-musl/release/avml acquire --compress output.lime
     - name: upload artifacts
       uses: actions/upload-artifact@v7.0.1
       with:
         name: linux-aarch64-artifacts
         path: |
           target/*/release/avml
           target/*/release/avml-minimal
-          target/*/release/avml-convert
-          target/*/release/avml-upload
   windows:
     permissions:
         contents: read
@@ -79,18 +75,14 @@ jobs:
         git config --global core.eol lf
     - uses: actions/checkout@v6.0.2
     - uses: Swatinem/rust-cache@65012b490220f477f20ab979e35ae732e6de4e68 # v2
-    - name: build avml-convert
-      run: cargo build --release --bin avml-convert --locked
-    - name: build avml-upload
-      run: cargo build --release --bin avml-upload --locked
+    - name: build avml
+      run: cargo build --release --bin avml --locked
     - name: Run tests
       run: cargo test
     - name: upload artifacts
       uses: actions/upload-artifact@v7.0.1
       with:
         name: windows-artifacts
         path: |
-          target/release/avml-convert.exe
-          target/release/avml_convert.pdb
-          target/release/avml-upload.exe
-          target/release/avml_upload.pdb
+          target/release/avml.exe
+          target/release/avml.pdb

Cargo.toml

@@ -13,11 +13,14 @@ edition = "2024"
 rust-version = "1.88.0"
 
 [features]
-default = ["put", "blobstore", "native-tls"]
+default = ["stream", "upload", "convert", "native-tls"]
 put = ["dep:reqwest", "reqwest?/stream", "dep:url", "dep:tokio", "dep:tokio-util", "dep:futures"]
-blobstore = ["dep:url", "dep:azure_core", "dep:azure_storage_blob", "dep:tokio", "dep:async-trait", "dep:futures"]
+blobstore = ["dep:url", "dep:azure_core", "dep:azure_storage_blob", "dep:tokio", "dep:async-trait", "dep:futures", "dep:tokio-util", "tokio/sync", "tokio-util/io-util"]
 status = ["dep:indicatif"]
 native-tls = ["dep:native-tls", "azure_core?/reqwest", "reqwest?/native-tls-vendored"]
+convert = []
+upload = ["put", "blobstore"]
+stream = ["blobstore", "tokio/net"]
 
 [dependencies]
 async-trait = {version="0.1", optional=true}
@@ -34,7 +37,7 @@ azure_storage_blob = {version="1.0", optional=true, default-features=false, feat
 indicatif = {version="0.18", optional=true, default-features=false}
 native-tls = {version="0.2", features=["vendored"], optional=true, default-features=false}
 reqwest = {version="0.13", optional=true, default-features=false}
-tokio = {version="1.52", default-features=false, optional=true, features=["fs", "rt-multi-thread", "io-util", "macros"]}
+tokio = {version="1.52", default-features=false, optional=true, features=["fs", "rt-multi-thread", "io-util", "macros", "net"]}
 tokio-util = {version="0.7", features=["codec"], optional=true, default-features=false}
 url = {version="2.5", optional=true, default-features=false}
 
@@ -69,5 +72,5 @@ panic="abort"
 codegen-units=1
 
 [[bin]]
-name = "avml-upload"
-required-features = ["put", "blobstore"]
+name = "avml"
+path = "src/bin/avml/main.rs"

README.md

@@ -32,22 +32,36 @@ If the memory source is not specified on the commandline, AVML will iterate over
 * Oracle Linux: 6.8, 6.9, 6.10, 7.3, 7.4, 7.5, 7.6, 7.9, 8.5, 9.0
 * [CBL-Mariner](https://github.com/microsoft/CBL-Mariner): 1.0, 2.0
 
+## Subcommands
+
+`avml` is a single binary with subcommands. Each subcommand is gated by
+a Cargo feature so a minimal build only includes the capability you need:
+
+| Subcommand | Feature   | Default | What it does                                                   |
+|------------|-----------|---------|----------------------------------------------------------------|
+| `acquire`  | (always)  | yes     | Snapshot memory to a local file (optional upload after).       |
+| `convert`  | `convert` | yes     | Convert between AVML / LiME / raw formats.                     |
+| `upload`   | `upload`  | yes     | Upload a local file via HTTP PUT or to Azure Block Blob.       |
+| `stream`   | `stream`  | yes     | Stream a snapshot directly to a destination, no local file.    |
+
+Build a minimal acquire-only binary with `cargo build --release --no-default-features`.
+
 # Getting Started
 
 ## Capturing a compressed memory image
 
 On the target host:
 
 ```
-avml --compress output.lime.compressed
+avml acquire --compress output.lime.compressed
 ```
 
 ## Capturing an uncompressed memory image
 
 On the target host:
 
 ```
-avml output.lime
+avml acquire output.lime
 ```
 
 ## Capturing a memory image & uploading to Azure Blob Store
@@ -60,7 +74,58 @@ SAS_URL=$(az storage blob generate-sas --account-name ACCOUNT --container CONTAI
 
 On the target host, execute avml with the generated SAS token.
 ```
-avml --sas-url ${SAS_URL} --delete output.lime
+avml acquire --sas-url ${SAS_URL} --delete output.lime
+```
+
+## Streaming a memory image without writing to local disk
+
+For hosts where writing the snapshot to a local file first is undesirable
+(read-only root, limited disk, forensic chain-of-custody concerns), use
+the `stream` subcommand. It picks the memory source once up front (same
+preference order as `acquire`'s `/dev/stdout` path — `/proc/kcore`, then
+`/dev/crash`, then `/dev/mem`; pass `--source` to override) and writes
+bytes sequentially to the chosen destination. The source cannot be
+changed mid-stream, so there is no automatic source fallback.
+
+### To Azure Block Blob Storage
+
+```
+avml stream blob ${SAS_URL}
+```
+
+- The block size is derived automatically so the snapshot fits within
+  Azure's per-blob 50,000-block limit. `--sas-block-size` (MiB) acts as
+  a *floor*; if the derived minimum is larger, the larger value wins.
+- `--sas-block-concurrency` caps the number of in-flight `stage_block`
+  calls. Peak RAM is approximately `(concurrency + 1) * block_size`.
+- If the snapshot fails mid-upload, staged blocks are abandoned without
+  being committed; Azure discards them automatically per its standard
+  policy.
+
+### To a remote TCP listener
+
+On the collector host:
+
+```
+nc -l 9000 > snapshot.lime
+```
+
+On the target host:
+
+```
+avml stream tcp collector.example.com:9000
+```
+
+avml connects once and writes the snapshot sequentially. If the
+connection drops mid-stream, the snapshot aborts; there is no resume.
+No TLS — pair with an SSH tunnel or stunnel for confidentiality and
+integrity if needed.
+
+## Uploading a previously-captured snapshot
+
+```
+avml upload put  ./output.lime ${URL}              # HTTP PUT
+avml upload blob ./output.lime ${SAS_URL}          # Azure Block Blob
 ```
 
 ## Capturing a memory image of an Azure VM using VM Extensions
@@ -71,7 +136,7 @@ On a secure host with `az cli` credentials, do the following:
 2. Create `config.json` containing the following information:
 ```
 {
-    "commandToExecute": "./avml --compress --sas-url <GENERATED_SAS_URL> --delete",
+    "commandToExecute": "./avml acquire --compress --sas-url <GENERATED_SAS_URL> --delete",
     "fileUris": ["https://FULL.URL.TO.AVML.example.com/avml"]
 }
 ```
@@ -85,73 +150,36 @@ On a secure host, generate a [S3 pre-signed URL](https://docs.aws.amazon.com/cli
 
 On the target host, execute avml with the generated pre-signed URL.
 ```
-avml --put ${URL} --delete output.lime
+avml acquire --url ${URL} --delete output.lime
 ```
 
 ## To decompress an AVML-compressed image
 ```
-avml-convert ./compressed.lime ./uncompressed.lime
+avml convert ./compressed.lime ./uncompressed.lime
 ```
 
 ## To compress an uncompressed LiME image
 ```
-avml-convert --source-format lime --format lime_compressed ./uncompressed.lime ./compressed.lime
+avml convert --source-format lime --format lime_compressed ./uncompressed.lime ./compressed.lime
 ```
 
 # Usage
 
 ```
-A portable volatile memory acquisition tool
-
-Usage: avml [OPTIONS] <FILENAME>
-
-Arguments:
-  <FILENAME>
-          name of the file to write to on local system
-
-Options:
-      --compress
-          compress via snappy
-
-      --source <SOURCE>
-          specify input source
+A portable volatile memory acquisition tool for Linux
 
-          Possible values:
-          - /dev/crash:
-            Provides a read-only view of physical memory.  Access to memory using this device must be paged aligned and read one page at a time
-          - /dev/mem:
-            Provides a read-write view of physical memory, though AVML opens it in a read-only fashion.  Access to to memory using this device can be disabled using the kernel configuration options `CONFIG_STRICT_DEVMEM` or `CONFIG_IO_STRICT_DEVMEM`
-          - /proc/kcore:
-            Provides a virtual ELF coredump of kernel memory.  This can be used to access physical memory
+Usage: avml <COMMAND>
 
-      --max-disk-usage <MAX_DISK_USAGE>
-          Specify the maximum estimated disk usage (in MB)
-
-      --max-disk-usage-percentage <MAX_DISK_USAGE_PERCENTAGE>
-          Specify the maximum estimated disk usage to stay under
-
-      --url <URL>
-          upload via HTTP PUT upon acquisition
-
-      --delete
-          delete upon successful upload
-
-      --sas-url <SAS_URL>
-          upload via Azure Blob Store upon acquisition
-
-      --sas-block-size <SAS_BLOCK_SIZE>
-          specify maximum block size in MiB; must be greater than 0
-
-      --sas-block-concurrency <SAS_BLOCK_CONCURRENCY>
-          specify blob upload concurrency; must be greater than 0
-
-  -h, --help
-          Print help (see a summary with '-h')
-
-  -V, --version
-          Print version
+Commands:
+  acquire  Acquire a memory snapshot to a local file (and optionally upload it)
+  convert  Convert between AVML and LiME snapshot formats and a raw memory image
+  upload   Upload an already-acquired snapshot file to remote storage
+  stream   Stream a memory snapshot directly to remote storage, without writing it to a local file
+  help     Print this message or the help of the given subcommand(s)
 ```
 
+Run `avml <COMMAND> --help` for per-command options.
+
 # Building on Ubuntu
 
     # Install MUSL

eng/build.sh

@@ -18,8 +18,5 @@ done
 cargo +stable build --release --no-default-features --target ${ARCH}-unknown-linux-musl --locked
 cp target/${ARCH}-unknown-linux-musl/release/avml target/${ARCH}-unknown-linux-musl/release/avml-minimal
 cargo +stable build --release --target ${ARCH}-unknown-linux-musl --locked
-cargo +stable build --release --target ${ARCH}-unknown-linux-musl --locked --bin avml-upload --features "put blobstore status"
 strip target/${ARCH}-unknown-linux-musl/release/avml
 strip target/${ARCH}-unknown-linux-musl/release/avml-minimal
-strip target/${ARCH}-unknown-linux-musl/release/avml-convert
-strip target/${ARCH}-unknown-linux-musl/release/avml-upload

eng/test-azure-image.sh

@@ -45,6 +45,6 @@ IP=$(az vm create -g ${GROUP} --size ${SIZE} -n ${VM} --image ${SKU} --public-ip
 ssh-keygen -R ${IP} 2>/dev/null > /dev/null
 quiet scp -oStrictHostKeyChecking=no ${EXE} ${IP}:./avml
 quiet ssh -oStrictHostKeyChecking=no ${IP} sudo chmod +x avml
-quiet ssh -oStrictHostKeyChecking=no ${IP} sudo ./avml --compress /mnt/image.lime
+quiet ssh -oStrictHostKeyChecking=no ${IP} sudo ./avml acquire --compress /mnt/image.lime
 quiet ssh -oStrictHostKeyChecking=no ${IP} sudo chmod a+r /mnt/image.lime
 quiet scp -oStrictHostKeyChecking=no ${IP}:/mnt/image.lime ./${SKU}.lime