fix(bootstrap): keep the token valid until the vault is bootstrapped

This commit is contained in:
CleverWild
2026-09-07 17:40:15 +02:00
parent 5ade05d475
commit 2b02d4a9b1
13 changed files with 700 additions and 109 deletions

View File

@@ -1,4 +1,4 @@
use super::common::ChannelTransport;
use super::common::{ChannelTransport, spawn_actors};
use arbiter_crypto::authn::{self, AuthChallenge, SigningContext};
use arbiter_proto::{
ClientMetadata,
@@ -93,7 +93,7 @@ async fn insert_bootstrap_sentinel_operator(db: &db::DatabasePool) {
async fn spawn_test_actors(db: &db::DatabasePool) -> GlobalActors {
insert_bootstrap_sentinel_operator(db).await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap {

View File

@@ -24,6 +24,36 @@ pub(crate) async fn bootstrapped_vault(db: &db::DatabasePool) -> Vault {
actor
}
/// Spawns a full `GlobalActors` for a test, backing `Bootstrapper`'s token file with a
/// throwaway temp directory rather than the real `~/.arbiter` -- a test must never be able to
/// reach, let alone write to, the developer's real bootstrap token file.
pub(crate) async fn spawn_actors(db: db::DatabasePool) -> GlobalActors {
let home = tempfile::tempdir().expect("failed to create a temp home directory for a test");
GlobalActors::spawn_in(db, home.path())
.await
.expect("failed to spawn GlobalActors for a test")
}
/// Retries `probe` until it yields a value, then returns it.
///
/// Effects that travel over the message bus are not visible the moment the publishing call
/// returns: `Publish` only enqueues to the bus's mailbox, which then delivers to each
/// subscriber's mailbox in turn. A test observing such an effect waits for it instead of
/// reading straight after the call that triggered it.
pub(crate) async fn eventually<T, F, Fut>(what: &str, mut probe: F) -> T
where
F: FnMut() -> Fut,
Fut: Future<Output = Option<T>>,
{
for _ in 0..100 {
if let Some(value) = probe().await {
return value;
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
panic!("{what} did not happen within 2s");
}
pub(crate) async fn root_key_history_id(db: &db::DatabasePool) -> i32 {
let mut conn = db.get().await.unwrap();
let id = schema::arbiter_settings::table

View File

@@ -46,6 +46,16 @@ where
panic!("{what} did not happen within 2s");
}
/// Spawns a full `GlobalActors` for a test, backing `Bootstrapper`'s token file with a
/// throwaway temp directory rather than the real `~/.arbiter` -- a test must never be able to
/// reach, let alone write to, the developer's real bootstrap token file.
async fn spawn_actors(db: db::DatabasePool) -> GlobalActors {
let home = tempfile::tempdir().expect("failed to create a temp home directory for a test");
GlobalActors::spawn_in(db, home.path())
.await
.expect("failed to spawn GlobalActors for a test")
}
async fn register_operator(db: &db::DatabasePool, pubkey: &authn::PublicKey) -> OperatorIdentityId {
let mut conn = db.get().await.unwrap();
insert_into(operator_identity::table)
@@ -142,7 +152,7 @@ async fn insert_unapproved_client(db: &db::DatabasePool, pubkey: &authn::PublicK
#[tokio::test]
async fn create_proposal_returns_id() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap {
@@ -172,7 +182,7 @@ async fn create_proposal_returns_id() {
#[tokio::test]
async fn create_proposal_caps_the_ttl() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap {
@@ -213,7 +223,7 @@ async fn create_proposal_caps_the_ttl() {
#[tokio::test]
async fn single_operator_vote_reaches_quorum() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -258,7 +268,7 @@ async fn single_operator_vote_reaches_quorum() {
#[tokio::test]
async fn two_operator_first_vote_is_pending() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -304,7 +314,7 @@ async fn two_operator_first_vote_is_pending() {
#[tokio::test]
async fn duplicate_vote_rejected() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -365,7 +375,7 @@ async fn duplicate_vote_rejected() {
#[tokio::test]
async fn invalid_signature_rejected() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -406,7 +416,7 @@ async fn invalid_signature_rejected() {
#[tokio::test]
async fn query_pending_reports_a_tally_per_proposal() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap {
@@ -500,7 +510,7 @@ async fn query_pending_reports_a_tally_per_proposal() {
#[tokio::test]
async fn query_pending_excludes_already_voted() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -566,7 +576,7 @@ async fn query_pending_excludes_already_voted() {
#[tokio::test]
async fn expired_proposal_is_hidden_and_unvotable() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -624,7 +634,7 @@ async fn approve_sdk_client_writes_integrity_envelope() {
use arbiter_server::db::schema::integrity_envelope;
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -683,7 +693,7 @@ async fn approve_sdk_client_writes_integrity_envelope() {
#[tokio::test]
async fn grant_wallet_access_on_quorum_approval() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -744,7 +754,7 @@ async fn grant_wallet_access_on_quorum_approval() {
#[tokio::test]
async fn approve_persistent_grant_creates_basic_grant_row() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -841,7 +851,7 @@ async fn approve_one_off_transaction_stores_result() {
use chrono::Duration;
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -961,7 +971,7 @@ async fn approve_one_off_transaction_stores_result() {
#[tokio::test]
async fn replace_operator_updates_pubkey_and_starts_rekey() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -1033,7 +1043,7 @@ async fn replace_operator_updates_pubkey_and_starts_rekey() {
#[tokio::test]
async fn trigger_rekey_reaches_quorum() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -1075,7 +1085,7 @@ async fn trigger_rekey_reaches_quorum() {
async fn key_rotation_requires_full_quorum() {
// §3.3: ReplaceOperator needs all 3 operators to approve, not just shamir_threshold(3)=2
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) })
@@ -1129,7 +1139,7 @@ async fn key_rotation_requires_full_quorum() {
#[tokio::test]
async fn recovery_vote_rejected_when_sleeping() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors.vault.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) }).await.unwrap();
let op_key = authn::SigningKey::generate();
@@ -1175,7 +1185,7 @@ async fn recovery_vote_rejected_when_sleeping() {
#[tokio::test]
async fn recovery_vote_blocked_on_non_replace_proposal() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors.vault.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) }).await.unwrap();
let op_key = authn::SigningKey::generate();
@@ -1224,7 +1234,7 @@ async fn recovery_vote_blocked_on_non_replace_proposal() {
#[tokio::test]
async fn recovery_wakeup_can_be_cancelled() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors.vault.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) }).await.unwrap();
let key = authn::SigningKey::generate();
@@ -1253,7 +1263,7 @@ async fn recovery_wakeup_can_be_cancelled() {
#[tokio::test]
async fn recovery_wakeup_prevents_duplicate_request() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors.vault.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) }).await.unwrap();
let key = authn::SigningKey::generate();
@@ -1281,7 +1291,7 @@ async fn recovery_wakeup_prevents_duplicate_request() {
async fn recovery_operator_vote_contributes_to_replace_quorum() {
// 1 ordinary operator + 1 recovery operator; replace_operator needs both.
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors.vault.ask(Bootstrap { seal_key: KeyCell::from([0u8; 32]) }).await.unwrap();
let op_key = authn::SigningKey::generate();

View File

@@ -1,8 +1,11 @@
use super::common::ChannelTransport;
use super::common::{ChannelTransport, bootstrapped_vault, eventually, spawn_actors};
use arbiter_crypto::authn::{self, AuthChallenge, SigningContext};
use arbiter_proto::transport::{Error as TransportError, Receiver, Sender};
use arbiter_server::{
actors::{GlobalActors, bootstrap::GetToken, vault::Bootstrap},
actors::{
bootstrap::GetToken,
vault::{self, Bootstrap},
},
crypto::integrity,
db::{self, schema},
peers::operator::{self, Credentials, OperatorConnection, auth, vault_gate},
@@ -150,14 +153,7 @@ impl Sender<auth::Inbound> for StartTestTransport {
#[test_log::test]
pub async fn bootstrap_token_auth() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
actors
.vault
.ask(Bootstrap {
seal_key: arbiter_server::crypto::KeyCell::from([0u8; 32]),
})
.await
.unwrap();
let actors = spawn_actors(db.clone()).await;
let token = actors.bootstrapper.ask(GetToken).await.unwrap().unwrap();
let (mut server_transport, mut test_transport) = ChannelTransport::new();
@@ -211,11 +207,131 @@ pub async fn bootstrap_token_auth() {
assert_eq!(stored_pubkey, verifying_key(&new_key).encode().0.to_vec());
}
/// A multi-operator committee must all register with the same bootstrap token before bootstrap
/// completes, so verifying the token must not consume it. This is the reachability bug fixed by
/// replacing `consume_token` with `verify_token`.
#[tokio::test]
#[test_log::test]
pub async fn bootstrap_invalid_token_auth() {
pub async fn bootstrap_token_registers_every_committee_member() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
let token = actors.bootstrapper.ask(GetToken).await.unwrap().unwrap();
for _ in 0..2 {
let (mut server_transport, mut test_transport) = ChannelTransport::new();
let db_for_task = db.clone();
let actors_for_task = actors.clone();
let task = tokio::spawn(async move {
let mut props = OperatorConnection::new(db_for_task, actors_for_task);
auth::authenticate(&mut props, &mut server_transport).await
});
let new_key = MlDsa87::key_gen(&mut rand::rng());
test_transport
.send(auth::Inbound::AuthChallengeRequest {
pubkey: verifying_key(&new_key).into(),
bootstrap_token: Some(token.clone()),
})
.await
.unwrap();
let response = test_transport
.recv()
.await
.expect("should receive challenge");
let challenge = match response {
Ok(auth::Outbound::AuthChallenge { challenge }) => challenge,
other => panic!("Expected AuthChallenge, got {other:?}"),
};
let signature = sign_operator_challenge(&new_key, &challenge);
test_transport
.send(auth::Inbound::AuthChallengeSolution {
signature: signature.to_bytes(),
})
.await
.unwrap();
let response = test_transport
.recv()
.await
.expect("should receive auth result");
assert!(matches!(response, Ok(auth::Outbound::AuthSuccess)));
task.await.unwrap().unwrap();
}
let mut conn = db.get().await.unwrap();
let registered: i64 = schema::operator_identity::table
.count()
.get_result(&mut conn)
.await
.unwrap();
assert_eq!(registered, 2);
// Bootstrap has not completed: the token must still be valid.
assert_eq!(
actors.bootstrapper.ask(GetToken).await.unwrap(),
Some(token)
);
}
/// `GlobalActors` must subscribe `Bootstrapper` to `events::Bootstrapped` on the message bus.
/// Without that one registration the token stays valid forever in production, which is the
/// defect this task exists to fix, and no other test notices: the test below drives the event
/// handler directly, and the `challenge_auth` family never re-reads the token after
/// bootstrapping. This one goes the whole way round -- real `Vault::bootstrap`, real bus --
/// and waits for the effect rather than reading straight after the call, because `Publish`
/// only enqueues to the bus's mailbox.
#[tokio::test]
#[test_log::test]
pub async fn bootstrapped_event_retires_the_token_through_the_message_bus() {
let db = db::create_test_pool().await;
let actors = spawn_actors(db.clone()).await;
assert!(
actors.bootstrapper.ask(GetToken).await.unwrap().is_some(),
"the token must exist before the vault is bootstrapped"
);
actors
.vault
.ask(Bootstrap {
seal_key: arbiter_server::crypto::KeyCell::from([0u8; 32]),
})
.await
.unwrap();
eventually("the bootstrap token to be retired", || async {
actors
.bootstrapper
.ask(GetToken)
.await
.unwrap()
.is_none()
.then_some(())
})
.await;
}
/// Once the vault reports `Bootstrapped`, the token is retired: further registrations must be
/// rejected even with a token that verified successfully moments earlier.
#[tokio::test]
#[test_log::test]
pub async fn bootstrap_token_rejected_after_bootstrapped_event() {
let db = db::create_test_pool().await;
let actors = spawn_actors(db.clone()).await;
let token = actors.bootstrapper.ask(GetToken).await.unwrap().unwrap();
// Drive the Bootstrapper's own event handler directly rather than through
// `actors.vault.ask(Bootstrap { .. })` + the message bus: bus delivery is fire-and-forget,
// so asserting on it would be racy. The handler under test is the same either way.
actors
.bootstrapper
.ask(vault::events::Bootstrapped)
.await
.unwrap();
assert!(actors.bootstrapper.ask(GetToken).await.unwrap().is_none());
let (mut server_transport, mut test_transport) = ChannelTransport::new();
let db_for_task = db.clone();
@@ -228,7 +344,7 @@ pub async fn bootstrap_invalid_token_auth() {
test_transport
.send(auth::Inbound::AuthChallengeRequest {
pubkey: verifying_key(&new_key).into(),
bootstrap_token: Some("invalid_token".to_owned()),
bootstrap_token: Some(token),
})
.await
.unwrap();
@@ -264,11 +380,150 @@ pub async fn bootstrap_invalid_token_auth() {
assert_eq!(count, 0);
}
/// `register_key`'s database gate must refuse a registration once `arbiter_settings.root_key_id`
/// is set, even when `Bootstrapper`'s own in-memory token has not yet been retired -- exactly
/// the two-mailbox-hop window between `Vault::bootstrap`'s commit and the `Bootstrapped` event
/// reaching `Bootstrapper` in production. "database bootstrapped, Bootstrapper not yet notified"
/// is reproduced deterministically by bootstrapping a throwaway `Vault` wired to its own message
/// bus: it commits `root_key_id` in the same database without ever publishing to the bus
/// `actors.bootstrapper` is registered on, so `actors.bootstrapper`'s token is left untouched.
#[tokio::test]
#[test_log::test]
pub async fn bootstrap_token_rejected_once_the_database_is_bootstrapped() {
let db = db::create_test_pool().await;
let actors = spawn_actors(db.clone()).await;
let token = actors.bootstrapper.ask(GetToken).await.unwrap().unwrap();
bootstrapped_vault(&db).await;
// From Bootstrapper's point of view the token still verifies: it never received an event.
assert_eq!(
actors.bootstrapper.ask(GetToken).await.unwrap(),
Some(token.clone())
);
let (mut server_transport, mut test_transport) = ChannelTransport::new();
let db_for_task = db.clone();
let task = tokio::spawn(async move {
let mut props = OperatorConnection::new(db_for_task, actors);
auth::authenticate(&mut props, &mut server_transport).await
});
let new_key = MlDsa87::key_gen(&mut rand::rng());
test_transport
.send(auth::Inbound::AuthChallengeRequest {
pubkey: verifying_key(&new_key).into(),
bootstrap_token: Some(token),
})
.await
.unwrap();
let response = test_transport
.recv()
.await
.expect("should receive challenge");
let challenge = match response {
Ok(auth::Outbound::AuthChallenge { challenge }) => challenge,
other => panic!("Expected AuthChallenge, got {other:?}"),
};
let signature = sign_operator_challenge(&new_key, &challenge);
test_transport
.send(auth::Inbound::AuthChallengeSolution {
signature: signature.to_bytes(),
})
.await
.unwrap();
// The refusal has to reach the peer, not just the task's return value: a registration
// refused after the database was bootstrapped must look like the refusal of a token that
// never verified, rather than a handshake that stops with nothing on the wire.
let refusal = test_transport
.recv()
.await
.expect("the refusal must be sent to the peer");
assert!(matches!(refusal, Err(auth::Error::InvalidBootstrapToken)));
assert!(matches!(
task.await.unwrap(),
Err(auth::Error::InvalidBootstrapToken)
));
let mut conn = db.get().await.unwrap();
let count: i64 = schema::operator_identity::table
.count()
.get_result::<i64>(&mut conn)
.await
.unwrap();
assert_eq!(count, 0);
}
#[tokio::test]
#[test_log::test]
pub async fn bootstrap_invalid_token_auth() {
let db = db::create_test_pool().await;
let actors = spawn_actors(db.clone()).await;
let (mut server_transport, mut test_transport) = ChannelTransport::new();
let db_for_task = db.clone();
let task = tokio::spawn(async move {
let mut props = OperatorConnection::new(db_for_task, actors);
auth::authenticate(&mut props, &mut server_transport).await
});
let new_key = MlDsa87::key_gen(&mut rand::rng());
test_transport
.send(auth::Inbound::AuthChallengeRequest {
pubkey: verifying_key(&new_key).into(),
bootstrap_token: Some("invalid_token".to_owned()),
})
.await
.unwrap();
let response = test_transport
.recv()
.await
.expect("should receive challenge");
let challenge = match response {
Ok(auth::Outbound::AuthChallenge { challenge }) => challenge,
other => panic!("Expected AuthChallenge, got {other:?}"),
};
let signature = sign_operator_challenge(&new_key, &challenge);
test_transport
.send(auth::Inbound::AuthChallengeSolution {
signature: signature.to_bytes(),
})
.await
.unwrap();
// The reference behaviour the refusal above has to match: a token that never verified is
// reported to the peer. Pinned here so the two refusal paths cannot drift apart again.
let refusal = test_transport
.recv()
.await
.expect("the refusal must be sent to the peer");
assert!(matches!(refusal, Err(auth::Error::InvalidBootstrapToken)));
assert!(matches!(
task.await.unwrap(),
Err(auth::Error::InvalidBootstrapToken)
));
let mut conn = db.get().await.unwrap();
let count: i64 = schema::operator_identity::table
.count()
.get_result::<i64>(&mut conn)
.await
.unwrap();
assert_eq!(count, 0);
}
#[tokio::test]
#[test_log::test]
pub async fn challenge_auth() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap {
@@ -353,7 +608,7 @@ pub async fn challenge_auth() {
#[test_log::test]
pub async fn challenge_auth_rejects_integrity_tag_mismatch_when_unsealed() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
@@ -427,7 +682,7 @@ pub async fn challenge_auth_rejects_integrity_tag_mismatch_when_unsealed() {
#[test_log::test]
pub async fn challenge_auth_rejects_invalid_signature() {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault
.ask(Bootstrap {

View File

@@ -1,9 +1,7 @@
use super::common::spawn_actors;
use arbiter_crypto::authn;
use arbiter_server::{
actors::{
GlobalActors,
vault::{Bootstrap, Seal},
},
actors::vault::{Bootstrap, Seal},
db,
peers::operator::{
Credentials,
@@ -26,7 +24,7 @@ async fn setup_sealed_gate(
oneshot::Receiver<Result<(), VaultGateError>>,
) {
let db = db::create_test_pool().await;
let actors = GlobalActors::spawn(db.clone()).await.unwrap();
let actors = spawn_actors(db.clone()).await;
actors
.vault