refactor(broker): split the Registry by responsibility
registry.go keeps the state and construction. Membership mutation, probe scheduling, snapshot publication and the observation writer move to their own files without behaviour changes.
- commit
- aebe61cf800bd7066ddfae88c6e7146116336d1e
- author
- brice · · 2026-10-08 05:12 UTC
- parent
- fbbc30c
- signature
- GPG Verified
CD88F9E564E0102646FB6077A1A07C7C7C3E349D· arch-gpg Verified for the committer email
6 files · +1209 -1163
M internal/usecase/broker/doc.go +7 -0
| @@ -2,4 +2,11 @@ | |||
| 2 | 2 | // reachability and catalogue snapshots, pooled physical transports, and | |
| 3 | 3 | // multiplexed logical streams. It does not own picker UI, sessions, PTYs, | |
| 4 | 4 | // terminal geometry, or attachment semantics. | |
| 5 | + | // | |
| 6 | + | // Registry state lives in registry.go and is guarded by one mutex. Its | |
| 7 | + | // behaviour is split by responsibility: durable membership mutation | |
| 8 | + | // (registry_membership.go), probe scheduling and result application | |
| 9 | + | // (registry_probe.go), local daemon observation (registry_local.go), | |
| 10 | + | // snapshot publication and subscriptions (registry_publish.go), and the | |
| 11 | + | // asynchronous observation writer (registry_snapshot_writer.go). | |
| 5 | 12 | package broker | |
| @@ -2,4 +2,11 @@ | |||||
| 2 | // reachability and catalogue snapshots, pooled physical transports, and | 2 | // reachability and catalogue snapshots, pooled physical transports, and | ||
| 3 | // multiplexed logical streams. It does not own picker UI, sessions, PTYs, | 3 | // multiplexed logical streams. It does not own picker UI, sessions, PTYs, | ||
| 4 | // terminal geometry, or attachment semantics. | 4 | // terminal geometry, or attachment semantics. | ||
| 5 | + | // | |||
| 6 | + | // Registry state lives in registry.go and is guarded by one mutex. Its | |||
| 7 | + | // behaviour is split by responsibility: durable membership mutation | |||
| 8 | + | // (registry_membership.go), probe scheduling and result application | |||
| 9 | + | // (registry_probe.go), local daemon observation (registry_local.go), | |||
| 10 | + | // snapshot publication and subscriptions (registry_publish.go), and the | |||
| 11 | + | // asynchronous observation writer (registry_snapshot_writer.go). | |||
| 5 | package broker | 12 | package broker | ||
M internal/usecase/broker/registry.go +0 -1163
| @@ -3,14 +3,10 @@ package broker | |||
| 3 | 3 | import ( | |
| 4 | 4 | "context" | |
| 5 | 5 | "crypto/rand" | |
| 6 | - | "encoding/binary" | |
| 7 | 6 | "errors" | |
| 8 | 7 | "fmt" | |
| 9 | - | "hash/fnv" | |
| 10 | 8 | "log/slog" | |
| 11 | 9 | "reflect" | |
| 12 | - | "slices" | |
| 13 | - | "sort" | |
| 14 | 10 | "sync" | |
| 15 | 11 | "sync/atomic" | |
| 16 | 12 | "time" | |
| @@ -424,1162 +420,3 @@ func (r *Registry) restore(snapshot ports.BrokerSnapshot) error { | |||
| 424 | 420 | } | |
| 425 | 421 | return nil | |
| 426 | 422 | } | |
| 427 | - | ||
| 428 | - | // Snapshot returns a defensive immutable copy without I/O. | |
| 429 | - | func (r *Registry) Snapshot() ports.BrokerSnapshot { return r.current.Load().Clone() } | |
| 430 | - | ||
| 431 | - | // Subscribe returns a capacity-one notification stream. Slow consumers never | |
| 432 | - | // block publication and always re-read the newest snapshot. | |
| 433 | - | func (r *Registry) Subscribe() ports.BrokerSubscription { | |
| 434 | - | s := &subscription{changed: make(chan struct{}, 1)} | |
| 435 | - | s.close = func() { | |
| 436 | - | r.mu.Lock() | |
| 437 | - | delete(r.subs, s) | |
| 438 | - | r.mu.Unlock() | |
| 439 | - | } | |
| 440 | - | r.mu.Lock() | |
| 441 | - | r.subs[s] = struct{}{} | |
| 442 | - | r.mu.Unlock() | |
| 443 | - | s.offer() | |
| 444 | - | return s | |
| 445 | - | } | |
| 446 | - | ||
| 447 | - | // AddHost adds or pins endpoint under policy. Duplicate same-policy additions | |
| 448 | - | // CAS-verify authority and preserve identity; a conflicting policy is refused. | |
| 449 | - | func (r *Registry) LookupHost(ctx context.Context, endpoint string) (ports.BrokerHostRecord, bool, error) { | |
| 450 | - | if err := ctx.Err(); err != nil { | |
| 451 | - | return ports.BrokerHostRecord{}, false, err | |
| 452 | - | } | |
| 453 | - | r.mu.Lock() | |
| 454 | - | defer r.mu.Unlock() | |
| 455 | - | for _, record := range r.authority.Hosts { | |
| 456 | - | if record.Registration.Endpoint == endpoint { | |
| 457 | - | record.Route = record.Route.Clone() | |
| 458 | - | return record, true, nil | |
| 459 | - | } | |
| 460 | - | } | |
| 461 | - | return ports.BrokerHostRecord{}, false, nil | |
| 462 | - | } | |
| 463 | - | ||
| 464 | - | func (r *Registry) AddHost(ctx context.Context, endpoint string, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |
| 465 | - | if err := domain.ValidateRemoteHostTarget(endpoint); err != nil { | |
| 466 | - | return domain.RemoteRegistration{}, err | |
| 467 | - | } | |
| 468 | - | if err := policy.Validate(); err != nil { | |
| 469 | - | return domain.RemoteRegistration{}, err | |
| 470 | - | } | |
| 471 | - | r.mu.Lock() | |
| 472 | - | defer r.mu.Unlock() | |
| 473 | - | if err := r.mutableLocked(ctx); err != nil { | |
| 474 | - | return domain.RemoteRegistration{}, err | |
| 475 | - | } | |
| 476 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 477 | - | for i := range records { | |
| 478 | - | if records[i].Registration.Endpoint != endpoint { | |
| 479 | - | continue | |
| 480 | - | } | |
| 481 | - | if records[i].Policy != policy { | |
| 482 | - | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 483 | - | } | |
| 484 | - | records[i].Pinned = true | |
| 485 | - | if err := r.commitMembershipLocked(records); err != nil { | |
| 486 | - | return domain.RemoteRegistration{}, err | |
| 487 | - | } | |
| 488 | - | return records[i].Registration, nil | |
| 489 | - | } | |
| 490 | - | id, err := r.newIncarnation() | |
| 491 | - | if err != nil { | |
| 492 | - | return domain.RemoteRegistration{}, err | |
| 493 | - | } | |
| 494 | - | reg, err := domain.NewRemoteRegistration(endpoint, id) | |
| 495 | - | if err != nil { | |
| 496 | - | return domain.RemoteRegistration{}, err | |
| 497 | - | } | |
| 498 | - | if reg.IsZero() { | |
| 499 | - | return domain.RemoteRegistration{}, errors.New("broker: incarnation generator returned zero") | |
| 500 | - | } | |
| 501 | - | route, err := ports.BrokerRouteForTransport(policy.Transport, endpoint) | |
| 502 | - | if err != nil { | |
| 503 | - | return domain.RemoteRegistration{}, err | |
| 504 | - | } | |
| 505 | - | records = append(records, ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy, Route: route}) | |
| 506 | - | if err := r.commitMembershipLocked(records); err != nil { | |
| 507 | - | return domain.RemoteRegistration{}, err | |
| 508 | - | } | |
| 509 | - | return reg, nil | |
| 510 | - | } | |
| 511 | - | ||
| 512 | - | // RemoveHost removes only the exact expected registration. | |
| 513 | - | // BindAuthenticatedIdentity durably binds first-contact identity under the | |
| 514 | - | // exact registration and policy fence. The same binding is idempotent; a | |
| 515 | - | // different identity is refused and never overwrites authority. | |
| 516 | - | func (r *Registry) BindAuthenticatedIdentity(ctx context.Context, request ports.BrokerIdentityBindingRequest) (ports.BrokerDaemonIdentity, error) { | |
| 517 | - | if err := request.Validate(); err != nil { | |
| 518 | - | return "", err | |
| 519 | - | } | |
| 520 | - | if request.Fence.Local { | |
| 521 | - | return request.Identity, nil | |
| 522 | - | } | |
| 523 | - | if err := ctx.Err(); err != nil { | |
| 524 | - | return "", err | |
| 525 | - | } | |
| 526 | - | r.mu.Lock() | |
| 527 | - | defer r.mu.Unlock() | |
| 528 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 529 | - | for index := range records { | |
| 530 | - | record := &records[index] | |
| 531 | - | if record.Identity == request.Identity && record.Policy != request.Policy { | |
| 532 | - | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity is already bound under an incompatible policy")} | |
| 533 | - | } | |
| 534 | - | } | |
| 535 | - | for index := range records { | |
| 536 | - | record := &records[index] | |
| 537 | - | if !record.Registration.Equal(request.Fence.Registration) || record.Policy != request.Policy { | |
| 538 | - | continue | |
| 539 | - | } | |
| 540 | - | if record.Identity != "" { | |
| 541 | - | if record.Identity != request.Identity { | |
| 542 | - | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity conflicts with durable binding")} | |
| 543 | - | } | |
| 544 | - | return record.Identity, nil | |
| 545 | - | } | |
| 546 | - | record.Identity = request.Identity | |
| 547 | - | if err := r.commitMembershipLocked(records); err != nil { | |
| 548 | - | return "", err | |
| 549 | - | } | |
| 550 | - | return request.Identity, nil | |
| 551 | - | } | |
| 552 | - | return "", ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: errors.New("broker: stale identity binding fence")} | |
| 553 | - | } | |
| 554 | - | ||
| 555 | - | func (r *Registry) RemoveHost(ctx context.Context, expected domain.RemoteRegistration) (bool, error) { | |
| 556 | - | if err := expected.Validate(); err != nil { | |
| 557 | - | return false, err | |
| 558 | - | } | |
| 559 | - | r.mu.Lock() | |
| 560 | - | defer r.mu.Unlock() | |
| 561 | - | if err := r.mutableLocked(ctx); err != nil { | |
| 562 | - | return false, err | |
| 563 | - | } | |
| 564 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 565 | - | for i := range records { | |
| 566 | - | if records[i].Registration.Endpoint != expected.Endpoint { | |
| 567 | - | continue | |
| 568 | - | } | |
| 569 | - | if !records[i].Registration.Equal(expected) { | |
| 570 | - | return false, ports.ErrBrokerHostConflict | |
| 571 | - | } | |
| 572 | - | records = append(records[:i], records[i+1:]...) | |
| 573 | - | if err := r.commitMembershipLocked(records); err != nil { | |
| 574 | - | return false, err | |
| 575 | - | } | |
| 576 | - | return true, nil | |
| 577 | - | } | |
| 578 | - | // A no-op still verifies the loaded revision. | |
| 579 | - | return false, r.commitMembershipLocked(records) | |
| 580 | - | } | |
| 581 | - | ||
| 582 | - | // UpdateHostPolicy updates exact authority and advances its generation. | |
| 583 | - | func (r *Registry) UpdateHostPolicy(ctx context.Context, expected domain.RemoteRegistration, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |
| 584 | - | if err := expected.Validate(); err != nil { | |
| 585 | - | return domain.RemoteRegistration{}, err | |
| 586 | - | } | |
| 587 | - | if err := policy.Validate(); err != nil { | |
| 588 | - | return domain.RemoteRegistration{}, err | |
| 589 | - | } | |
| 590 | - | r.mu.Lock() | |
| 591 | - | defer r.mu.Unlock() | |
| 592 | - | if err := r.mutableLocked(ctx); err != nil { | |
| 593 | - | return domain.RemoteRegistration{}, err | |
| 594 | - | } | |
| 595 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 596 | - | for i := range records { | |
| 597 | - | if records[i].Registration.Endpoint != expected.Endpoint { | |
| 598 | - | continue | |
| 599 | - | } | |
| 600 | - | if !records[i].Registration.Equal(expected) { | |
| 601 | - | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 602 | - | } | |
| 603 | - | if records[i].Policy == policy { | |
| 604 | - | if err := r.commitMembershipLocked(records); err != nil { | |
| 605 | - | return domain.RemoteRegistration{}, err | |
| 606 | - | } | |
| 607 | - | return expected, nil | |
| 608 | - | } | |
| 609 | - | if expected.Generation == ^domain.RemoteGeneration(0) { | |
| 610 | - | return domain.RemoteRegistration{}, ports.ErrBrokerRevisionExhausted | |
| 611 | - | } | |
| 612 | - | route, err := ports.BrokerRouteForTransport(policy.Transport, expected.Endpoint) | |
| 613 | - | if err != nil { | |
| 614 | - | return domain.RemoteRegistration{}, err | |
| 615 | - | } | |
| 616 | - | records[i].Registration.Generation++ | |
| 617 | - | records[i].Route = route | |
| 618 | - | records[i].Policy = policy | |
| 619 | - | if err := r.commitMembershipLocked(records); err != nil { | |
| 620 | - | return domain.RemoteRegistration{}, err | |
| 621 | - | } | |
| 622 | - | return records[i].Registration, nil | |
| 623 | - | } | |
| 624 | - | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 625 | - | } | |
| 626 | - | ||
| 627 | - | func (r *Registry) mutableLocked(ctx context.Context) error { | |
| 628 | - | if r.membershipMode != MembershipMutable { | |
| 629 | - | return ports.ErrBrokerMembershipImmutable | |
| 630 | - | } | |
| 631 | - | if err := ctx.Err(); err != nil { | |
| 632 | - | return err | |
| 633 | - | } | |
| 634 | - | if r.running.Load() == runStopped { | |
| 635 | - | return ports.ErrBrokerRegistryClosed | |
| 636 | - | } | |
| 637 | - | if r.revisionExhaustedLocked() { | |
| 638 | - | return ports.ErrBrokerRevisionExhausted | |
| 639 | - | } | |
| 640 | - | return nil | |
| 641 | - | } | |
| 642 | - | ||
| 643 | - | // commitMembershipLocked is the sole mutable-membership durable commit path. | |
| 644 | - | // It installs projection only after CAS success. | |
| 645 | - | func (r *Registry) commitMembershipLocked(records []ports.BrokerHostRecord) error { | |
| 646 | - | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 647 | - | return err | |
| 648 | - | } | |
| 649 | - | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | |
| 650 | - | return errors.New("broker: remote hosts require a host probe") | |
| 651 | - | } | |
| 652 | - | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | |
| 653 | - | order := make([]string, 0, len(records)) | |
| 654 | - | for rank, record := range records { | |
| 655 | - | observation := stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |
| 656 | - | // A membership the registry cannot publish is refused here, before the | |
| 657 | - | // store CAS, rather than persisted and left to freeze every later | |
| 658 | - | // publication: an endpoint whose derived display hint is empty after the | |
| 659 | - | // display rule drops its runes makes every snapshot carrying it invalid. | |
| 660 | - | if err := observation.Validate(); err != nil { | |
| 661 | - | return fmt.Errorf("broker: host record %q is not publishable: %w", record.Registration.Endpoint, err) | |
| 662 | - | } | |
| 663 | - | next[record.Registration.Endpoint] = observation | |
| 664 | - | order = append(order, record.Registration.Endpoint) | |
| 665 | - | } | |
| 666 | - | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | |
| 667 | - | return fmt.Errorf("broker: replace hosts: %w", err) | |
| 668 | - | } | |
| 669 | - | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: ports.CloneBrokerHostRecords(records)} | |
| 670 | - | r.setHostsLocked(next, order) | |
| 671 | - | return nil | |
| 672 | - | } | |
| 673 | - | ||
| 674 | - | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |
| 675 | - | // authority revision. It delegates to commitMembershipLocked, the same | |
| 676 | - | // validated commit path the runtime mutations use, so membership whose | |
| 677 | - | // projection could never be published is refused before the store CAS instead | |
| 678 | - | // of being persisted and silently freezing every later publication. Success | |
| 679 | - | // means membership is committed, not that advisory observations have flushed. | |
| 680 | - | // Store errors (including stale authority) are returned without changing the | |
| 681 | - | // projection, probe attempts, or local token. A settled run reports | |
| 682 | - | // ports.ErrBrokerRegistryClosed and an exhausted revision series reports | |
| 683 | - | // ports.ErrBrokerRevisionExhausted; both refuse the mutation before the store | |
| 684 | - | // CAS, so a refusal never changes durable authority either. Conflicts are not | |
| 685 | - | // retried or refreshed implicitly: reconcile by reopening the owner. Callers | |
| 686 | - | // must supply fresh incarnations when removing and re-adding. Even identical | |
| 687 | - | // records perform CAS so stale authority cannot report success. The synchronous | |
| 688 | - | // write holds mu; Snapshot remains lock-free, but scheduling waits for | |
| 689 | - | // membership durability. Observation writes remain asynchronous. | |
| 690 | - | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |
| 691 | - | var err error | |
| 692 | - | records, err = ports.UpgradeBrokerHostRoutes(records) | |
| 693 | - | if err != nil { | |
| 694 | - | return err | |
| 695 | - | } | |
| 696 | - | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 697 | - | return err | |
| 698 | - | } | |
| 699 | - | r.mu.Lock() | |
| 700 | - | defer r.mu.Unlock() | |
| 701 | - | if r.running.Load() == runStopped { | |
| 702 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |
| 703 | - | } | |
| 704 | - | if r.revisionExhaustedLocked() { | |
| 705 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |
| 706 | - | } | |
| 707 | - | return r.commitMembershipLocked(records) | |
| 708 | - | } | |
| 709 | - | ||
| 710 | - | // setHostsLocked applies only a projection; the public path commits authority | |
| 711 | - | // first. The caller holds mu through both steps, fencing probe completions. | |
| 712 | - | // order is the registration order the projection was built from; it becomes | |
| 713 | - | // the publication order of remote daemons. | |
| 714 | - | func (r *Registry) setHostsLocked(next map[string]ports.BrokerDaemonObservation, order []string) { | |
| 715 | - | if registrationsUnchanged(r.hosts, next, r.order, order) { | |
| 716 | - | return | |
| 717 | - | } | |
| 718 | - | for endpoint, current := range r.hosts { | |
| 719 | - | candidate, present := next[endpoint] | |
| 720 | - | if present && current.Registration.Equal(candidate.Registration) && current.Policy == candidate.Policy { | |
| 721 | - | // Unchanged authority: keep the live observation as is. Rank is the | |
| 722 | - | // one authority field that follows membership order rather than the | |
| 723 | - | // record, so it is re-stamped from the new projection. | |
| 724 | - | kept := current | |
| 725 | - | kept.Rank = candidate.Rank | |
| 726 | - | next[endpoint] = kept | |
| 727 | - | continue | |
| 728 | - | } | |
| 729 | - | // A retired registration owns no observation: cancelling its attempt | |
| 730 | - | // releases a ctx-honoring probe immediately, and dropping its record | |
| 731 | - | // means the retired completion can never clear or answer the | |
| 732 | - | // replacement, and the replacement is never blocked behind it. The | |
| 733 | - | // cancellation happens under the registry lock, so no replacement | |
| 734 | - | // attempt can start before the retired context is cancelled. | |
| 735 | - | r.cancelInflightLocked(endpoint) | |
| 736 | - | delete(r.pending, endpoint) | |
| 737 | - | if present { | |
| 738 | - | // The endpoint stays live under a new registration, so its | |
| 739 | - | // retirement is never published as a tombstone alongside a live | |
| 740 | - | // host; the attempt token fences the stale completion. | |
| 741 | - | delete(r.tombstones, endpoint) | |
| 742 | - | continue | |
| 743 | - | } | |
| 744 | - | r.retireLocked(current.Registration) | |
| 745 | - | } | |
| 746 | - | for endpoint := range next { | |
| 747 | - | // Re-adding an endpoint supersedes its tombstone: a live host and a | |
| 748 | - | // tombstone for the same endpoint are mutually exclusive. | |
| 749 | - | delete(r.tombstones, endpoint) | |
| 750 | - | } | |
| 751 | - | r.hosts = next | |
| 752 | - | r.order = order | |
| 753 | - | r.publishLocked(true) | |
| 754 | - | r.hint() | |
| 755 | - | } | |
| 756 | - | ||
| 757 | - | // cancelInflightLocked cancels the in-flight attempt for one endpoint and | |
| 758 | - | // drops its record. Callers must hold r.mu: the retired attempt's context is | |
| 759 | - | // cancelled before the record disappears, so no replacement attempt can start | |
| 760 | - | // while the retired probe's context is still live. | |
| 761 | - | func (r *Registry) cancelInflightLocked(endpoint string) { | |
| 762 | - | if attempt := r.inflight[endpoint]; attempt != nil { | |
| 763 | - | attempt.cancel() | |
| 764 | - | delete(r.inflight, endpoint) | |
| 765 | - | } | |
| 766 | - | } | |
| 767 | - | ||
| 768 | - | // cancelAllInflightLocked cancels every in-flight attempt and drops the | |
| 769 | - | // records. Callers must hold r.mu. | |
| 770 | - | func (r *Registry) cancelAllInflightLocked() { | |
| 771 | - | for endpoint, attempt := range r.inflight { | |
| 772 | - | attempt.cancel() | |
| 773 | - | delete(r.inflight, endpoint) | |
| 774 | - | } | |
| 775 | - | } | |
| 776 | - | ||
| 777 | - | // RequestProbe is non-blocking and coalesces concurrent demand per endpoint. | |
| 778 | - | // A disabled registry never observes, so the demand is dropped without | |
| 779 | - | // scheduling anything. An empty endpoint requests a local re-probe instead of a | |
| 780 | - | // configured remote: the broker's own machine daemon is re-observed ahead of | |
| 781 | - | // its scheduled freshness window, honored by dispatchLocalLocked exactly like | |
| 782 | - | // pending honors remote demand. A registry with no local observation producer | |
| 783 | - | // drops it, just like an unknown remote endpoint is dropped. | |
| 784 | - | func (r *Registry) RequestProbe(endpoint string) { | |
| 785 | - | if r.observationDisabled { | |
| 786 | - | return | |
| 787 | - | } | |
| 788 | - | r.mu.Lock() | |
| 789 | - | if endpoint == "" { | |
| 790 | - | if r.local != nil { | |
| 791 | - | r.localPending = true | |
| 792 | - | } | |
| 793 | - | } else if _, ok := r.hosts[endpoint]; ok { | |
| 794 | - | r.pending[endpoint] = true | |
| 795 | - | } | |
| 796 | - | r.mu.Unlock() | |
| 797 | - | r.hint() | |
| 798 | - | } | |
| 799 | - | ||
| 800 | - | // SetDemand records one live client subscription opening (active=true) or | |
| 801 | - | // closing (active=false). internal/usecase/broker/service.go calls it from | |
| 802 | - | // Service.Subscribe and serviceSubscription.Close, so the demand count tracks | |
| 803 | - | // exactly the connections a client is actively watching. The first | |
| 804 | - | // subscription requests one immediate observation per host; while demand | |
| 805 | - | // stays positive, the demand cadence schedules later observations. With no | |
| 806 | - | // subscription, only explicit RequestProbe calls dispatch. A disabled registry | |
| 807 | - | // still counts subscriptions but never observes. | |
| 808 | - | func (r *Registry) SetDemand(active bool) { | |
| 809 | - | r.mu.Lock() | |
| 810 | - | if active { | |
| 811 | - | if r.demand == 0 { | |
| 812 | - | r.localPending = r.local != nil | |
| 813 | - | for endpoint := range r.hosts { | |
| 814 | - | r.pending[endpoint] = true | |
| 815 | - | } | |
| 816 | - | } | |
| 817 | - | r.demand++ | |
| 818 | - | } else if r.demand > 0 { | |
| 819 | - | r.demand-- | |
| 820 | - | } | |
| 821 | - | r.mu.Unlock() | |
| 822 | - | r.hint() | |
| 823 | - | } | |
| 824 | - | ||
| 825 | - | // freshForLocked returns the demand cadence while subscribed. An absent | |
| 826 | - | // daemon uses the longer freshFor interval instead. Callers must hold r.mu. | |
| 827 | - | func (r *Registry) freshForLocked(local bool) time.Duration { | |
| 828 | - | if r.demand <= 0 { | |
| 829 | - | return r.freshFor | |
| 830 | - | } | |
| 831 | - | if local { | |
| 832 | - | return r.demandFreshForLocal | |
| 833 | - | } | |
| 834 | - | return r.demandFreshForRemote | |
| 835 | - | } | |
| 836 | - | ||
| 837 | - | func (r *Registry) hint() { | |
| 838 | - | select { | |
| 839 | - | case r.wake <- struct{}{}: | |
| 840 | - | default: | |
| 841 | - | } | |
| 842 | - | } | |
| 843 | - | ||
| 844 | - | // Run schedules observations until cancellation. Run owns the registry | |
| 845 | - | // lifecycle and may execute exactly once: a second call is refused so a | |
| 846 | - | // concurrent or restarted caller can never duplicate scheduling. On return it | |
| 847 | - | // clears in-flight observation state, publishes the settled projection, and | |
| 848 | - | // flushes durable persistence before any further write is refused. | |
| 849 | - | // | |
| 850 | - | // An observation-disabled registry runs the same lifecycle and drain but never | |
| 851 | - | // schedules observation: it arms no timer, starts no probe, and drops every | |
| 852 | - | // reconcile. It still owns and drains its durable writer, so a shutdown never | |
| 853 | - | // returns before the newest staged publication has reached the store. | |
| 854 | - | func (r *Registry) Run(ctx context.Context) { | |
| 855 | - | if !r.running.CompareAndSwap(runIdle, runActive) { | |
| 856 | - | r.log.Warn("broker: registry Run refused; single-run already started or finished") | |
| 857 | - | return | |
| 858 | - | } | |
| 859 | - | defer r.settle() | |
| 860 | - | if r.observationDisabled { | |
| 861 | - | <-ctx.Done() | |
| 862 | - | return | |
| 863 | - | } | |
| 864 | - | timer := r.clock.NewTimer(0) | |
| 865 | - | defer timer.Stop() | |
| 866 | - | results := make(chan probeResult, ports.BrokerMaxHosts) | |
| 867 | - | for { | |
| 868 | - | now := r.clock.Now() | |
| 869 | - | r.dispatch(ctx, now, results) | |
| 870 | - | timer.Reset(r.nextDelay(now)) | |
| 871 | - | select { | |
| 872 | - | case <-ctx.Done(): | |
| 873 | - | return | |
| 874 | - | case <-r.wake: | |
| 875 | - | case <-timer.C(): | |
| 876 | - | case result := <-results: | |
| 877 | - | if ctx.Err() != nil { | |
| 878 | - | // Shutdown wins: a cancellation is never recorded as an | |
| 879 | - | // observation failure. | |
| 880 | - | return | |
| 881 | - | } | |
| 882 | - | r.apply(result) | |
| 883 | - | } | |
| 884 | - | } | |
| 885 | - | } | |
| 886 | - | ||
| 887 | - | // settle is Run's cleanup: nothing stays in flight, the published projection | |
| 888 | - | // carries no transient Checking flag, durable persistence is flushed, and | |
| 889 | - | // every later Run call is refused. | |
| 890 | - | func (r *Registry) settle() { | |
| 891 | - | r.mu.Lock() | |
| 892 | - | r.cancelAllInflightLocked() | |
| 893 | - | if r.localAttempt != nil { | |
| 894 | - | r.localAttempt.cancel() | |
| 895 | - | r.localAttempt = nil | |
| 896 | - | } | |
| 897 | - | cleared := false | |
| 898 | - | if r.local != nil && r.localHost.Checking { | |
| 899 | - | r.localHost.Checking = false | |
| 900 | - | cleared = true | |
| 901 | - | } | |
| 902 | - | for endpoint, host := range r.hosts { | |
| 903 | - | if !host.Checking { | |
| 904 | - | continue | |
| 905 | - | } | |
| 906 | - | host.Checking = false | |
| 907 | - | r.hosts[endpoint] = host | |
| 908 | - | cleared = true | |
| 909 | - | } | |
| 910 | - | if cleared { | |
| 911 | - | r.publishLocked(true) | |
| 912 | - | } | |
| 913 | - | r.running.Store(runStopped) | |
| 914 | - | r.mu.Unlock() | |
| 915 | - | r.store.close() | |
| 916 | - | } | |
| 917 | - | ||
| 918 | - | // probeAttempt is one admitted observation: the registry-wide token that | |
| 919 | - | // fences its completion, and the cancel for the context derived from the run | |
| 920 | - | // context. Cancelling the derived context retires the attempt: a probe that | |
| 921 | - | // honors cancellation returns promptly, so a replaced or removed endpoint | |
| 922 | - | // never accumulates overlapping probes, and settle releases every attempt | |
| 923 | - | // before the run loop exits. | |
| 924 | - | type probeAttempt struct { | |
| 925 | - | token uint64 | |
| 926 | - | cancel context.CancelFunc | |
| 927 | - | } | |
| 928 | - | ||
| 929 | - | // probeResult is one completed observation bound to the exact attempt token | |
| 930 | - | // that started it. | |
| 931 | - | type probeResult struct { | |
| 932 | - | endpoint string | |
| 933 | - | registration domain.RemoteRegistration | |
| 934 | - | attempt uint64 | |
| 935 | - | snapshot ports.BrokerDaemonObservation | |
| 936 | - | err error | |
| 937 | - | at time.Time | |
| 938 | - | // local marks the result of one local probe attempt, which carries no | |
| 939 | - | // endpoint or registration and is applied to the configured local entry. | |
| 940 | - | local bool | |
| 941 | - | } | |
| 942 | - | ||
| 943 | - | func (r *Registry) dispatch(ctx context.Context, now time.Time, results chan<- probeResult) { | |
| 944 | - | r.mu.Lock() | |
| 945 | - | if r.demand == 0 && len(r.pending) == 0 && !r.localPending { | |
| 946 | - | r.mu.Unlock() | |
| 947 | - | return | |
| 948 | - | } | |
| 949 | - | started := false | |
| 950 | - | for endpoint, host := range r.hosts { | |
| 951 | - | if _, inflight := r.inflight[endpoint]; inflight { | |
| 952 | - | continue | |
| 953 | - | } | |
| 954 | - | explicit := r.pending[endpoint] | |
| 955 | - | if !explicit && (r.demand == 0 || !host.NextDue.IsZero() && now.Before(host.NextDue)) { | |
| 956 | - | continue | |
| 957 | - | } | |
| 958 | - | delete(r.pending, endpoint) | |
| 959 | - | r.attempts++ | |
| 960 | - | attempt := r.attempts | |
| 961 | - | probeCtx, cancel := context.WithCancel(ctx) | |
| 962 | - | r.inflight[endpoint] = &probeAttempt{token: attempt, cancel: cancel} | |
| 963 | - | if visibleCheck(host, explicit) { | |
| 964 | - | started = true | |
| 965 | - | host.Checking = true | |
| 966 | - | } | |
| 967 | - | host.LastAttempt = now | |
| 968 | - | r.hosts[endpoint] = host | |
| 969 | - | registration := host.Registration | |
| 970 | - | // The deadline is armed here, under r.mu and at dispatch time, so the | |
| 971 | - | // attempt's bound starts with the attempt rather than whenever its | |
| 972 | - | // goroutine is scheduled. | |
| 973 | - | deadline := r.clock.NewTimer(r.probeTimeout) | |
| 974 | - | go func(endpoint string, registration domain.RemoteRegistration, attempt uint64, probeCtx context.Context) { | |
| 975 | - | snapshot, err := observeBounded(probeCtx, deadline, func(ctx context.Context) (ports.BrokerDaemonObservation, error) { | |
| 976 | - | return r.probe.Probe(ctx, registration) | |
| 977 | - | }) | |
| 978 | - | result := probeResult{endpoint: endpoint, registration: registration, attempt: attempt, snapshot: snapshot, err: err, at: r.clock.Now()} | |
| 979 | - | select { | |
| 980 | - | case results <- result: | |
| 981 | - | case <-probeCtx.Done(): | |
| 982 | - | // The attempt was retired (replaced, removed, or shutdown), so | |
| 983 | - | // its completion is already fenced: dropping it keeps churn from | |
| 984 | - | // queueing results the run loop would only discard. | |
| 985 | - | } | |
| 986 | - | }(endpoint, registration, attempt, probeCtx) | |
| 987 | - | } | |
| 988 | - | // Only an observation that visibly started changes the projection; an | |
| 989 | - | // idle dispatch or a routine refresh must not bump the revision or wake | |
| 990 | - | // subscribers. The in-flight projection is never persisted: Checking is | |
| 991 | - | // transient. | |
| 992 | - | if r.dispatchLocalLocked(ctx, now, results) { | |
| 993 | - | started = true | |
| 994 | - | } | |
| 995 | - | if started { | |
| 996 | - | r.publishLocked(false) | |
| 997 | - | } | |
| 998 | - | r.mu.Unlock() | |
| 999 | - | } | |
| 1000 | - | ||
| 1001 | - | // visibleCheck reports whether starting an observation publishes the transient | |
| 1002 | - | // Checking state. A routine scheduled refresh of a reachable daemon stays | |
| 1003 | - | // silent until its result lands, so an idle watched broker publishes once per | |
| 1004 | - | // observation instead of twice; an explicit request or a daemon that is not | |
| 1005 | - | // confirmed reachable still shows that it is being checked. | |
| 1006 | - | func visibleCheck(host ports.BrokerDaemonObservation, explicit bool) bool { | |
| 1007 | - | return explicit || host.Availability != domain.RemoteAvailabilityReachable | |
| 1008 | - | } | |
| 1009 | - | ||
| 1010 | - | // observeBounded runs one observation under its attempt deadline. retire is the | |
| 1011 | - | // attempt's retirement context: a replaced, removed, or settled attempt returns | |
| 1012 | - | // at once and its completion is fenced by the caller. The probe itself runs on | |
| 1013 | - | // a context derived from retire that the deadline also cancels, so a | |
| 1014 | - | // ctx-honoring probe unwinds its dial, handshake, or exchange when either | |
| 1015 | - | // fires. The attempt never waits for the probe past its deadline: a probe that | |
| 1016 | - | // ignores cancellation is abandoned (its late result is discarded) and the | |
| 1017 | - | // attempt still reports errProbeTimeout, so the endpoint's single in-flight | |
| 1018 | - | // slot is always released. | |
| 1019 | - | func observeBounded(retire context.Context, deadline ports.Timer, observe func(context.Context) (ports.BrokerDaemonObservation, error)) (ports.BrokerDaemonObservation, error) { | |
| 1020 | - | defer deadline.Stop() | |
| 1021 | - | probeCtx, cancel := context.WithCancelCause(retire) | |
| 1022 | - | defer cancel(nil) | |
| 1023 | - | type outcome struct { | |
| 1024 | - | snapshot ports.BrokerDaemonObservation | |
| 1025 | - | err error | |
| 1026 | - | } | |
| 1027 | - | done := make(chan outcome, 1) | |
| 1028 | - | go func() { | |
| 1029 | - | snapshot, err := observe(probeCtx) | |
| 1030 | - | done <- outcome{snapshot: snapshot, err: err} | |
| 1031 | - | }() | |
| 1032 | - | select { | |
| 1033 | - | case result := <-done: | |
| 1034 | - | return result.snapshot, result.err | |
| 1035 | - | case <-deadline.C(): | |
| 1036 | - | cancel(errProbeTimeout) | |
| 1037 | - | return ports.BrokerDaemonObservation{}, errProbeTimeout | |
| 1038 | - | case <-retire.Done(): | |
| 1039 | - | return ports.BrokerDaemonObservation{}, retire.Err() | |
| 1040 | - | } | |
| 1041 | - | } | |
| 1042 | - | ||
| 1043 | - | func (r *Registry) apply(result probeResult) { | |
| 1044 | - | r.mu.Lock() | |
| 1045 | - | defer r.mu.Unlock() | |
| 1046 | - | if result.local { | |
| 1047 | - | r.applyLocal(result) | |
| 1048 | - | return | |
| 1049 | - | } | |
| 1050 | - | current, ok := r.hosts[result.endpoint] | |
| 1051 | - | attempt := r.inflight[result.endpoint] | |
| 1052 | - | if !ok || !current.Registration.Equal(result.registration) || attempt == nil || attempt.token != result.attempt { | |
| 1053 | - | // A stale completion touches neither current authority nor the | |
| 1054 | - | // in-flight slot, which may already belong to a newer registration or | |
| 1055 | - | // attempt. | |
| 1056 | - | return | |
| 1057 | - | } | |
| 1058 | - | delete(r.inflight, result.endpoint) | |
| 1059 | - | // The probe has returned, so releasing its derived context here only | |
| 1060 | - | // frees the attempt's resources; the record is gone before any new | |
| 1061 | - | // attempt can start. | |
| 1062 | - | attempt.cancel() | |
| 1063 | - | current.Checking = false | |
| 1064 | - | current.LastAttempt = result.at | |
| 1065 | - | ||
| 1066 | - | kind, availability, failed := observationOutcome(result) | |
| 1067 | - | if !failed && availability == domain.RemoteAvailabilityNoDaemon { | |
| 1068 | - | current.Availability = availability | |
| 1069 | - | current.LastSuccess = result.at | |
| 1070 | - | // An absent daemon is stable: demand does not require a fresh SSH | |
| 1071 | - | // handshake every two seconds just to confirm it is still absent. | |
| 1072 | - | current.NextDue = result.at.Add(r.jitter(r.freshFor, result.endpoint, result.attempt)) | |
| 1073 | - | current.ConsecutiveFailures = 0 | |
| 1074 | - | current.LastFailure = domain.RemoteFailure{} | |
| 1075 | - | current.Identity = "" | |
| 1076 | - | current.Incarnation = ports.BrokerDaemonIncarnation{} | |
| 1077 | - | current.ProtocolVersion = 0 | |
| 1078 | - | current.Capabilities = 0 | |
| 1079 | - | current.InventoryKnown = false | |
| 1080 | - | current.Sessions = nil | |
| 1081 | - | r.hosts[result.endpoint] = current | |
| 1082 | - | r.publishLocked(true) | |
| 1083 | - | return | |
| 1084 | - | } | |
| 1085 | - | if !failed { | |
| 1086 | - | // Configured authority comes only from the registry's own projection: | |
| 1087 | - | // a probe result supplies observed state, never endpoint, registration, | |
| 1088 | - | // policy, display origin, or rank. | |
| 1089 | - | observed := stampObservationAuthority(result.snapshot.Clone(), current) | |
| 1090 | - | observed.Checking = false | |
| 1091 | - | observed.LastAttempt = result.at | |
| 1092 | - | observed.LastSuccess = result.at | |
| 1093 | - | observed.NextDue = result.at.Add(r.jitter(r.freshForLocked(false), result.endpoint, result.attempt)) | |
| 1094 | - | observed.ConsecutiveFailures = 0 | |
| 1095 | - | observed.LastFailure = domain.RemoteFailure{} | |
| 1096 | - | observed.FailureEpisode = current.FailureEpisode | |
| 1097 | - | // The published observation must be fully valid once authority is | |
| 1098 | - | // stamped: a probe that invents partial identity (identity without a | |
| 1099 | - | // protocol version) or an inventory without a known catalogue is an | |
| 1100 | - | // invalid response, never a published observation. The durable rule is | |
| 1101 | - | // re-checked too: a projection the offline store would reject is never | |
| 1102 | - | // published as if it had persisted. The store applies the same rule. | |
| 1103 | - | if err := observed.Validate(); err != nil { | |
| 1104 | - | r.log.Debug("broker: observation is invalid", "endpoint", result.endpoint, "err", err) | |
| 1105 | - | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |
| 1106 | - | } else if err := ports.ValidateDurableHostProjection(observed); err != nil { | |
| 1107 | - | r.log.Debug("broker: observation is not persistable", "endpoint", result.endpoint, "err", err) | |
| 1108 | - | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |
| 1109 | - | } else { | |
| 1110 | - | r.hosts[result.endpoint] = observed | |
| 1111 | - | r.publishLocked(true) | |
| 1112 | - | return | |
| 1113 | - | } | |
| 1114 | - | } | |
| 1115 | - | // Every non-reachable outcome is a typed failure on the capped retry | |
| 1116 | - | // cadence: the last success, last-known inventory, and failure episode | |
| 1117 | - | // counter stay authoritative. | |
| 1118 | - | current.Availability = availability | |
| 1119 | - | if current.ConsecutiveFailures == 0 { | |
| 1120 | - | current.FailureEpisode++ | |
| 1121 | - | } | |
| 1122 | - | current.ConsecutiveFailures++ | |
| 1123 | - | current.LastFailure = domain.RemoteFailure{Kind: kind, Err: result.err} | |
| 1124 | - | delay := r.retryDelay(current.ConsecutiveFailures) | |
| 1125 | - | if r.demand > 0 { | |
| 1126 | - | delay = demandRetryDelay(r.demandFreshForRemote, current.ConsecutiveFailures) | |
| 1127 | - | } | |
| 1128 | - | current.NextDue = result.at.Add(r.jitter(delay, result.endpoint, result.attempt)) | |
| 1129 | - | r.log.Debug("broker_probe_failed", "endpoint", result.endpoint, "kind", kind, "failures", current.ConsecutiveFailures, "err", result.err, "cause", errors.Unwrap(result.err)) | |
| 1130 | - | r.hosts[result.endpoint] = current | |
| 1131 | - | r.publishLocked(true) | |
| 1132 | - | } | |
| 1133 | - | ||
| 1134 | - | // observationOutcome classifies a completed observation. It reports | |
| 1135 | - | // failed=false only for a confirmed reachable result; every other outcome | |
| 1136 | - | // carries the typed failure kind and the availability to publish. The caller | |
| 1137 | - | // additionally checks the adopted projection against the durable rules | |
| 1138 | - | // (ports.ValidateDurableHostProjection), which cover the session bound and | |
| 1139 | - | // catalogue validity for the exact incarnation and success time. | |
| 1140 | - | func observationOutcome(result probeResult) (domain.RemoteFailureKind, domain.RemoteAvailability, bool) { | |
| 1141 | - | if result.err != nil { | |
| 1142 | - | kind := domain.RemoteFailureTransport | |
| 1143 | - | var typed domain.RemoteFailure | |
| 1144 | - | var brokerErr ports.BrokerError | |
| 1145 | - | switch { | |
| 1146 | - | case errors.As(result.err, &typed) && typed.Kind != domain.RemoteFailureNone: | |
| 1147 | - | // The dialer classified the failure (for example SSH authentication). | |
| 1148 | - | kind = typed.Kind | |
| 1149 | - | case errors.As(result.err, &brokerErr) && (brokerErr.Code == ports.BrokerErrorConflictingPolicy || brokerErr.Code == ports.BrokerErrorIncompatible): | |
| 1150 | - | // SSH worked; the daemon's identity or version was refused. | |
| 1151 | - | kind = domain.RemoteFailureIncompatible | |
| 1152 | - | case errors.Is(result.err, context.DeadlineExceeded): | |
| 1153 | - | kind = domain.RemoteFailureTimeout | |
| 1154 | - | } | |
| 1155 | - | return kind, availabilityFor(kind), true | |
| 1156 | - | } | |
| 1157 | - | switch result.snapshot.Availability { | |
| 1158 | - | case domain.RemoteAvailabilityReachable: | |
| 1159 | - | return domain.RemoteFailureNone, domain.RemoteAvailabilityReachable, false | |
| 1160 | - | case domain.RemoteAvailabilityNoDaemon: | |
| 1161 | - | return domain.RemoteFailureNone, domain.RemoteAvailabilityNoDaemon, false | |
| 1162 | - | case domain.RemoteAvailabilityIncompatible: | |
| 1163 | - | return domain.RemoteFailureIncompatible, domain.RemoteAvailabilityIncompatible, true | |
| 1164 | - | case domain.RemoteAvailabilityAuthFailed: | |
| 1165 | - | return domain.RemoteFailureAuthentication, domain.RemoteAvailabilityAuthFailed, true | |
| 1166 | - | case domain.RemoteAvailabilityInvalidResponse: | |
| 1167 | - | return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |
| 1168 | - | default: | |
| 1169 | - | // Unknown or Unreachable without an error still means reachability was | |
| 1170 | - | // not confirmed, so it retries on the typed backoff cadence. | |
| 1171 | - | return domain.RemoteFailureTransport, domain.RemoteAvailabilityUnreachable, true | |
| 1172 | - | } | |
| 1173 | - | } | |
| 1174 | - | ||
| 1175 | - | // availabilityFor maps a typed failure kind to the published availability. | |
| 1176 | - | // The typed kind is preserved separately for notice policy. | |
| 1177 | - | func availabilityFor(kind domain.RemoteFailureKind) domain.RemoteAvailability { | |
| 1178 | - | switch kind { | |
| 1179 | - | case domain.RemoteFailureAuthentication: | |
| 1180 | - | return domain.RemoteAvailabilityAuthFailed | |
| 1181 | - | case domain.RemoteFailureIncompatible: | |
| 1182 | - | return domain.RemoteAvailabilityIncompatible | |
| 1183 | - | case domain.RemoteFailureInvalidResponse: | |
| 1184 | - | return domain.RemoteAvailabilityInvalidResponse | |
| 1185 | - | default: | |
| 1186 | - | return domain.RemoteAvailabilityUnreachable | |
| 1187 | - | } | |
| 1188 | - | } | |
| 1189 | - | ||
| 1190 | - | // demandRetryDelay is the watched-host failure backoff: the demand cadence | |
| 1191 | - | // doubled per consecutive failure, capped at demandRetryMax. | |
| 1192 | - | func demandRetryDelay(base time.Duration, failures uint) time.Duration { | |
| 1193 | - | delay := base | |
| 1194 | - | for i := uint(1); i < failures && delay < demandRetryMax; i++ { | |
| 1195 | - | delay *= 2 | |
| 1196 | - | } | |
| 1197 | - | return min(delay, max(base, demandRetryMax)) | |
| 1198 | - | } | |
| 1199 | - | ||
| 1200 | - | // retryDelay is the capped exponential retry cadence for a failure streak. | |
| 1201 | - | func (r *Registry) retryDelay(failures uint) time.Duration { | |
| 1202 | - | delay := r.retryBase | |
| 1203 | - | for i := uint(1); i < failures; i++ { | |
| 1204 | - | if delay >= r.retryMax { | |
| 1205 | - | return r.retryMax | |
| 1206 | - | } | |
| 1207 | - | delay *= 2 | |
| 1208 | - | } | |
| 1209 | - | if delay > r.retryMax { | |
| 1210 | - | return r.retryMax | |
| 1211 | - | } | |
| 1212 | - | return delay | |
| 1213 | - | } | |
| 1214 | - | ||
| 1215 | - | // jittered spreads a base interval deterministically across endpoint and | |
| 1216 | - | // attempt: identical inputs always yield the identical duration, and the | |
| 1217 | - | // result stays within 90%-110% of the base. No wall clock or global | |
| 1218 | - | // randomness participates, so retry and healthy-refresh schedules are | |
| 1219 | - | // reproducible instead of racing a shared PRNG. | |
| 1220 | - | func jittered(base time.Duration, endpoint string, attempt uint64) time.Duration { | |
| 1221 | - | if base <= 0 { | |
| 1222 | - | return base | |
| 1223 | - | } | |
| 1224 | - | h := fnv.New32a() | |
| 1225 | - | h.Write([]byte(endpoint)) | |
| 1226 | - | var counter [8]byte | |
| 1227 | - | binary.BigEndian.PutUint64(counter[:], attempt) | |
| 1228 | - | h.Write(counter[:]) | |
| 1229 | - | spread := int64(h.Sum32() % 21) | |
| 1230 | - | return base*90/100 + time.Duration(spread)*base/100 | |
| 1231 | - | } | |
| 1232 | - | ||
| 1233 | - | func (r *Registry) nextDelay(now time.Time) time.Duration { | |
| 1234 | - | r.mu.Lock() | |
| 1235 | - | defer r.mu.Unlock() | |
| 1236 | - | if r.demand == 0 { | |
| 1237 | - | if r.localPending || len(r.pending) > 0 { | |
| 1238 | - | return 0 | |
| 1239 | - | } | |
| 1240 | - | return time.Hour | |
| 1241 | - | } | |
| 1242 | - | var earliest time.Time | |
| 1243 | - | if r.local != nil && r.localAttempt == nil { | |
| 1244 | - | if r.localHost.NextDue.IsZero() { | |
| 1245 | - | return 0 | |
| 1246 | - | } | |
| 1247 | - | if earliest.IsZero() || r.localHost.NextDue.Before(earliest) { | |
| 1248 | - | earliest = r.localHost.NextDue | |
| 1249 | - | } | |
| 1250 | - | } | |
| 1251 | - | for endpoint, host := range r.hosts { | |
| 1252 | - | // An observation already in flight is completed by its result, never | |
| 1253 | - | // by the schedule; waiting on it here would spin the run loop. | |
| 1254 | - | if _, inflight := r.inflight[endpoint]; inflight { | |
| 1255 | - | continue | |
| 1256 | - | } | |
| 1257 | - | if r.pending[endpoint] || host.NextDue.IsZero() { | |
| 1258 | - | return 0 | |
| 1259 | - | } | |
| 1260 | - | if earliest.IsZero() || host.NextDue.Before(earliest) { | |
| 1261 | - | earliest = host.NextDue | |
| 1262 | - | } | |
| 1263 | - | } | |
| 1264 | - | if earliest.IsZero() { | |
| 1265 | - | return time.Hour | |
| 1266 | - | } | |
| 1267 | - | if delay := earliest.Sub(now); delay > 0 { | |
| 1268 | - | return delay | |
| 1269 | - | } | |
| 1270 | - | return 0 | |
| 1271 | - | } | |
| 1272 | - | ||
| 1273 | - | // registrationsUnchanged reports exact membership equality: the same endpoints | |
| 1274 | - | // in the same registration order, bound to the same registration identities and | |
| 1275 | - | // policy. Publication order and presentation rank are functions of durable | |
| 1276 | - | // membership order, so a reordering of identical records is a membership change | |
| 1277 | - | // that must republish; otherwise the same durable state yields different | |
| 1278 | - | // publications depending on process history. Projection fields such as | |
| 1279 | - | // availability, checking, or inventory never participate. | |
| 1280 | - | func registrationsUnchanged(current, next map[string]ports.BrokerDaemonObservation, currentOrder, nextOrder []string) bool { | |
| 1281 | - | if len(current) != len(next) { | |
| 1282 | - | return false | |
| 1283 | - | } | |
| 1284 | - | if !slices.Equal(currentOrder, nextOrder) { | |
| 1285 | - | return false | |
| 1286 | - | } | |
| 1287 | - | for endpoint, candidate := range next { | |
| 1288 | - | existing, ok := current[endpoint] | |
| 1289 | - | if !ok || !existing.Registration.Equal(candidate.Registration) || existing.Policy != candidate.Policy { | |
| 1290 | - | return false | |
| 1291 | - | } | |
| 1292 | - | } | |
| 1293 | - | return true | |
| 1294 | - | } | |
| 1295 | - | ||
| 1296 | - | // retireLocked records the tombstone for one removed registration at the | |
| 1297 | - | // revision of the publication that carries it. Mutation paths refuse an | |
| 1298 | - | // exhausted series before retiring, so the retirement revision can never wrap | |
| 1299 | - | // to zero or fall below the last published one. | |
| 1300 | - | func (r *Registry) retireLocked(registration domain.RemoteRegistration) { | |
| 1301 | - | retired := r.revision + 1 | |
| 1302 | - | r.tombstones[registration.Endpoint] = ports.BrokerHostTombstone{ | |
| 1303 | - | Endpoint: registration.Endpoint, | |
| 1304 | - | Registration: registration, | |
| 1305 | - | RetiredRevision: retired, | |
| 1306 | - | } | |
| 1307 | - | r.pruneTombstonesLocked() | |
| 1308 | - | } | |
| 1309 | - | ||
| 1310 | - | // revisionExhaustedLocked reports whether this epoch has no revision left to | |
| 1311 | - | // publish, latching the exhausted state on first detection. Callers must hold | |
| 1312 | - | // r.mu. | |
| 1313 | - | func (r *Registry) revisionExhaustedLocked() bool { | |
| 1314 | - | if r.revisionExhausted { | |
| 1315 | - | return true | |
| 1316 | - | } | |
| 1317 | - | if r.revision != ^ports.BrokerRevision(0) { | |
| 1318 | - | return false | |
| 1319 | - | } | |
| 1320 | - | r.revisionExhausted = true | |
| 1321 | - | r.log.Error("broker: revision series exhausted; refusing further publication", "epoch", r.epoch) | |
| 1322 | - | return true | |
| 1323 | - | } | |
| 1324 | - | ||
| 1325 | - | // pruneTombstonesLocked keeps only the newest BrokerMaxTombstones retirements | |
| 1326 | - | // so an endpoint that churns can never grow the published set without bound. | |
| 1327 | - | func (r *Registry) pruneTombstonesLocked() { | |
| 1328 | - | for len(r.tombstones) > ports.BrokerMaxTombstones { | |
| 1329 | - | oldest := "" | |
| 1330 | - | var oldestRevision ports.BrokerRevision | |
| 1331 | - | for endpoint, tombstone := range r.tombstones { | |
| 1332 | - | if oldest == "" || tombstone.RetiredRevision < oldestRevision || | |
| 1333 | - | (tombstone.RetiredRevision == oldestRevision && endpoint < oldest) { | |
| 1334 | - | oldest, oldestRevision = endpoint, tombstone.RetiredRevision | |
| 1335 | - | } | |
| 1336 | - | } | |
| 1337 | - | delete(r.tombstones, oldest) | |
| 1338 | - | } | |
| 1339 | - | } | |
| 1340 | - | ||
| 1341 | - | // tombstonesLocked renders the bounded tombstone publication, newest first. | |
| 1342 | - | // Live endpoints are never published as tombstones. | |
| 1343 | - | func (r *Registry) tombstonesLocked() []ports.BrokerHostTombstone { | |
| 1344 | - | if len(r.tombstones) == 0 { | |
| 1345 | - | return nil | |
| 1346 | - | } | |
| 1347 | - | out := make([]ports.BrokerHostTombstone, 0, len(r.tombstones)) | |
| 1348 | - | for endpoint, tombstone := range r.tombstones { | |
| 1349 | - | if _, live := r.hosts[endpoint]; live { | |
| 1350 | - | continue | |
| 1351 | - | } | |
| 1352 | - | out = append(out, tombstone) | |
| 1353 | - | } | |
| 1354 | - | sort.Slice(out, func(i, j int) bool { | |
| 1355 | - | if out[i].RetiredRevision != out[j].RetiredRevision { | |
| 1356 | - | return out[i].RetiredRevision > out[j].RetiredRevision | |
| 1357 | - | } | |
| 1358 | - | return out[i].Endpoint < out[j].Endpoint | |
| 1359 | - | }) | |
| 1360 | - | if len(out) > ports.BrokerMaxTombstones { | |
| 1361 | - | out = out[:ports.BrokerMaxTombstones] | |
| 1362 | - | } | |
| 1363 | - | return out | |
| 1364 | - | } | |
| 1365 | - | ||
| 1366 | - | func (r *Registry) publishLocked(persist bool) { | |
| 1367 | - | // Fail closed on an exhausted series: a wrapped revision would be zero, and | |
| 1368 | - | // a wrapped series would publish revisions below the newest one, either of | |
| 1369 | - | // which silently freezes durable persistence. The last valid snapshot stays | |
| 1370 | - | // published, nothing is stored, and no subscriber is woken for a publication | |
| 1371 | - | // that did not happen. | |
| 1372 | - | if r.revisionExhaustedLocked() { | |
| 1373 | - | return | |
| 1374 | - | } | |
| 1375 | - | // Remote daemons are published in registration order. The broker's own | |
| 1376 | - | // machine daemon, when a producer is configured, is the | |
| 1377 | - | // prepended entry at index zero; without a producer the registry produces no | |
| 1378 | - | // local observation and never invents one. | |
| 1379 | - | daemons := make([]ports.BrokerDaemonObservation, 0, len(r.hosts)+1) | |
| 1380 | - | if r.local != nil { | |
| 1381 | - | daemons = append(daemons, r.localHost.Clone()) | |
| 1382 | - | } | |
| 1383 | - | for _, endpoint := range r.order { | |
| 1384 | - | host, ok := r.hosts[endpoint] | |
| 1385 | - | if !ok { | |
| 1386 | - | continue | |
| 1387 | - | } | |
| 1388 | - | daemons = append(daemons, host.Clone()) | |
| 1389 | - | } | |
| 1390 | - | snapshot := ports.BrokerSnapshot{Epoch: r.epoch, Revision: r.revision + 1, Daemons: daemons, Removed: r.tombstonesLocked()} | |
| 1391 | - | if err := snapshot.Validate(); err != nil { | |
| 1392 | - | // Fail closed: a publication the wire refuses aborts every client | |
| 1393 | - | // connection on subscribe, so an invalid projection is never published. | |
| 1394 | - | // The last valid snapshot stays current, nothing is persisted, and the | |
| 1395 | - | // revision series does not advance past a publication that never | |
| 1396 | - | // happened. | |
| 1397 | - | r.log.Error("broker: refused invalid snapshot publication", "revision", r.revision+1, "err", err) | |
| 1398 | - | return | |
| 1399 | - | } | |
| 1400 | - | r.revision = snapshot.Revision | |
| 1401 | - | r.current.Store(&snapshot) | |
| 1402 | - | if persist { | |
| 1403 | - | // enqueue only stages an immutable publication: rendering the durable | |
| 1404 | - | // copy and every store call happen on the writer goroutine, never | |
| 1405 | - | // under the registry lock and never on the probe schedule. | |
| 1406 | - | r.store.enqueue(snapshot) | |
| 1407 | - | } | |
| 1408 | - | for sub := range r.subs { | |
| 1409 | - | sub.offer() | |
| 1410 | - | } | |
| 1411 | - | } | |
| 1412 | - | ||
| 1413 | - | // durable returns the persistable copy of a publication, rendered by the | |
| 1414 | - | // writer goroutine with the shared rule (ports.BrokerSnapshot.DurableProjection). | |
| 1415 | - | // The registry may publish the process-local daemon, which is never durable, | |
| 1416 | - | // so it is dropped first; the store would refuse it otherwise. Restored | |
| 1417 | - | // observations regain policy from membership (see stampHostAuthority). | |
| 1418 | - | func durable(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |
| 1419 | - | return snapshot.WithoutLocal().DurableProjection() | |
| 1420 | - | } | |
| 1421 | - | ||
| 1422 | - | // snapshotWriter serializes durable snapshot writes off the registry lock. | |
| 1423 | - | // Enqueue never blocks on store I/O and always coalesces to the newest | |
| 1424 | - | // revision; Store calls run on one goroutine, so the durable store never sees | |
| 1425 | - | // an out-of-order write. A failed write is logged and retried by the next | |
| 1426 | - | // publication, so a store fault never stalls publication. Close flushes the | |
| 1427 | - | // newest pending snapshot, stops the writer, and waits, so no write outlives | |
| 1428 | - | // shutdown. | |
| 1429 | - | type snapshotWriter struct { | |
| 1430 | - | store ports.BrokerSnapshotStore | |
| 1431 | - | log *slog.Logger | |
| 1432 | - | ||
| 1433 | - | mu sync.Mutex | |
| 1434 | - | pending *ports.BrokerSnapshot | |
| 1435 | - | written ports.BrokerRevision | |
| 1436 | - | // last is the durable content of the newest successful write with its | |
| 1437 | - | // revision and observation timestamps cleared. Only the writer goroutine | |
| 1438 | - | // touches it. | |
| 1439 | - | last *ports.BrokerSnapshot | |
| 1440 | - | started bool | |
| 1441 | - | closed bool | |
| 1442 | - | wake chan struct{} | |
| 1443 | - | stop chan struct{} | |
| 1444 | - | done chan struct{} | |
| 1445 | - | } | |
| 1446 | - | ||
| 1447 | - | func newSnapshotWriter(store ports.BrokerSnapshotStore, log *slog.Logger) *snapshotWriter { | |
| 1448 | - | return &snapshotWriter{ | |
| 1449 | - | store: store, log: log, | |
| 1450 | - | wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), | |
| 1451 | - | } | |
| 1452 | - | } | |
| 1453 | - | ||
| 1454 | - | // enqueue stages a snapshot for durable persistence. It coalesces to the | |
| 1455 | - | // newest revision, starts the writer on first use, and never blocks on the | |
| 1456 | - | // store or copies session payloads. Enqueue after Close is refused: nothing is | |
| 1457 | - | // written after shutdown. | |
| 1458 | - | func (w *snapshotWriter) enqueue(snapshot ports.BrokerSnapshot) { | |
| 1459 | - | w.mu.Lock() | |
| 1460 | - | if w.closed { | |
| 1461 | - | w.mu.Unlock() | |
| 1462 | - | w.log.Debug("broker: durable snapshot write refused after shutdown", "revision", snapshot.Revision) | |
| 1463 | - | return | |
| 1464 | - | } | |
| 1465 | - | if w.pending != nil { | |
| 1466 | - | if snapshot.Revision <= w.pending.Revision { | |
| 1467 | - | w.mu.Unlock() | |
| 1468 | - | return | |
| 1469 | - | } | |
| 1470 | - | } else if snapshot.Revision <= w.written { | |
| 1471 | - | w.mu.Unlock() | |
| 1472 | - | return | |
| 1473 | - | } | |
| 1474 | - | // The publication is immutable after publication, so staging it is a value | |
| 1475 | - | // copy; the defensive durable copy is rendered on the writer goroutine. | |
| 1476 | - | w.pending = &snapshot | |
| 1477 | - | if !w.started { | |
| 1478 | - | w.started = true | |
| 1479 | - | go w.run() | |
| 1480 | - | } | |
| 1481 | - | w.mu.Unlock() | |
| 1482 | - | select { | |
| 1483 | - | case w.wake <- struct{}{}: | |
| 1484 | - | default: | |
| 1485 | - | } | |
| 1486 | - | } | |
| 1487 | - | ||
| 1488 | - | func (w *snapshotWriter) run() { | |
| 1489 | - | defer close(w.done) | |
| 1490 | - | for { | |
| 1491 | - | select { | |
| 1492 | - | case <-w.stop: | |
| 1493 | - | w.flush() | |
| 1494 | - | return | |
| 1495 | - | case <-w.wake: | |
| 1496 | - | w.flush() | |
| 1497 | - | } | |
| 1498 | - | } | |
| 1499 | - | } | |
| 1500 | - | ||
| 1501 | - | // flush writes the newest staged snapshot, skipping revisions the store | |
| 1502 | - | // already holds. | |
| 1503 | - | func (w *snapshotWriter) flush() { | |
| 1504 | - | for { | |
| 1505 | - | w.mu.Lock() | |
| 1506 | - | snapshot := w.pending | |
| 1507 | - | written := w.written | |
| 1508 | - | w.pending = nil | |
| 1509 | - | w.mu.Unlock() | |
| 1510 | - | if snapshot == nil { | |
| 1511 | - | return | |
| 1512 | - | } | |
| 1513 | - | if snapshot.Revision <= written { | |
| 1514 | - | continue | |
| 1515 | - | } | |
| 1516 | - | next := durable(*snapshot) | |
| 1517 | - | key := durableContent(next) | |
| 1518 | - | if w.last != nil && reflect.DeepEqual(*w.last, key) { | |
| 1519 | - | // Only observation timestamps moved: the stored copy already | |
| 1520 | - | // carries the same durable state, so skip the rewrite and fsync. | |
| 1521 | - | // A restart re-observes every host before trusting freshness. | |
| 1522 | - | continue | |
| 1523 | - | } | |
| 1524 | - | if err := w.store.Store(next); err != nil { | |
| 1525 | - | w.last = nil | |
| 1526 | - | w.log.Error("broker: durable snapshot write failed", "revision", snapshot.Revision, "err", err) | |
| 1527 | - | continue | |
| 1528 | - | } | |
| 1529 | - | w.last = &key | |
| 1530 | - | w.mu.Lock() | |
| 1531 | - | if snapshot.Revision > w.written { | |
| 1532 | - | w.written = snapshot.Revision | |
| 1533 | - | } | |
| 1534 | - | w.mu.Unlock() | |
| 1535 | - | } | |
| 1536 | - | } | |
| 1537 | - | ||
| 1538 | - | // durableContent returns the durable projection without the fields that change | |
| 1539 | - | // on every routine observation: the revision and the attempt, success, and | |
| 1540 | - | // due timestamps. | |
| 1541 | - | func durableContent(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |
| 1542 | - | out := snapshot | |
| 1543 | - | out.Revision = 0 | |
| 1544 | - | out.Daemons = make([]ports.BrokerDaemonObservation, len(snapshot.Daemons)) | |
| 1545 | - | for i, daemon := range snapshot.Daemons { | |
| 1546 | - | daemon.LastAttempt = time.Time{} | |
| 1547 | - | daemon.LastSuccess = time.Time{} | |
| 1548 | - | daemon.NextDue = time.Time{} | |
| 1549 | - | out.Daemons[i] = daemon | |
| 1550 | - | } | |
| 1551 | - | return out | |
| 1552 | - | } | |
| 1553 | - | ||
| 1554 | - | // close flushes the newest staged snapshot, stops the writer, and waits for | |
| 1555 | - | // the drain to finish. It never returns before the in-flight write completes. | |
| 1556 | - | func (w *snapshotWriter) close() { | |
| 1557 | - | w.mu.Lock() | |
| 1558 | - | if w.closed { | |
| 1559 | - | w.mu.Unlock() | |
| 1560 | - | return | |
| 1561 | - | } | |
| 1562 | - | w.closed = true | |
| 1563 | - | started := w.started | |
| 1564 | - | w.mu.Unlock() | |
| 1565 | - | if !started { | |
| 1566 | - | return | |
| 1567 | - | } | |
| 1568 | - | close(w.stop) | |
| 1569 | - | <-w.done | |
| 1570 | - | } | |
| 1571 | - | ||
| 1572 | - | type subscription struct { | |
| 1573 | - | changed chan struct{} | |
| 1574 | - | once sync.Once | |
| 1575 | - | close func() | |
| 1576 | - | } | |
| 1577 | - | ||
| 1578 | - | func (s *subscription) Changed() <-chan struct{} { return s.changed } | |
| 1579 | - | func (s *subscription) Close() { s.once.Do(s.close) } | |
| 1580 | - | func (s *subscription) offer() { | |
| 1581 | - | select { | |
| 1582 | - | case s.changed <- struct{}{}: | |
| 1583 | - | default: | |
| 1584 | - | } | |
| 1585 | - | } | |
| @@ -3,14 +3,10 @@ package broker | |||||
| 3 | import ( | 3 | import ( | ||
| 4 | "context" | 4 | "context" | ||
| 5 | "crypto/rand" | 5 | "crypto/rand" | ||
| 6 | - | "encoding/binary" | |||
| 7 | "errors" | 6 | "errors" | ||
| 8 | "fmt" | 7 | "fmt" | ||
| 9 | - | "hash/fnv" | |||
| 10 | "log/slog" | 8 | "log/slog" | ||
| 11 | "reflect" | 9 | "reflect" | ||
| 12 | - | "slices" | |||
| 13 | - | "sort" | |||
| 14 | "sync" | 10 | "sync" | ||
| 15 | "sync/atomic" | 11 | "sync/atomic" | ||
| 16 | "time" | 12 | "time" | ||
| @@ -424,1162 +420,3 @@ func (r *Registry) restore(snapshot ports.BrokerSnapshot) error { | |||||
| 424 | } | 420 | } | ||
| 425 | return nil | 421 | return nil | ||
| 426 | } | 422 | } | ||
| 427 | - | ||||
| 428 | - | // Snapshot returns a defensive immutable copy without I/O. | |||
| 429 | - | func (r *Registry) Snapshot() ports.BrokerSnapshot { return r.current.Load().Clone() } | |||
| 430 | - | ||||
| 431 | - | // Subscribe returns a capacity-one notification stream. Slow consumers never | |||
| 432 | - | // block publication and always re-read the newest snapshot. | |||
| 433 | - | func (r *Registry) Subscribe() ports.BrokerSubscription { | |||
| 434 | - | s := &subscription{changed: make(chan struct{}, 1)} | |||
| 435 | - | s.close = func() { | |||
| 436 | - | r.mu.Lock() | |||
| 437 | - | delete(r.subs, s) | |||
| 438 | - | r.mu.Unlock() | |||
| 439 | - | } | |||
| 440 | - | r.mu.Lock() | |||
| 441 | - | r.subs[s] = struct{}{} | |||
| 442 | - | r.mu.Unlock() | |||
| 443 | - | s.offer() | |||
| 444 | - | return s | |||
| 445 | - | } | |||
| 446 | - | ||||
| 447 | - | // AddHost adds or pins endpoint under policy. Duplicate same-policy additions | |||
| 448 | - | // CAS-verify authority and preserve identity; a conflicting policy is refused. | |||
| 449 | - | func (r *Registry) LookupHost(ctx context.Context, endpoint string) (ports.BrokerHostRecord, bool, error) { | |||
| 450 | - | if err := ctx.Err(); err != nil { | |||
| 451 | - | return ports.BrokerHostRecord{}, false, err | |||
| 452 | - | } | |||
| 453 | - | r.mu.Lock() | |||
| 454 | - | defer r.mu.Unlock() | |||
| 455 | - | for _, record := range r.authority.Hosts { | |||
| 456 | - | if record.Registration.Endpoint == endpoint { | |||
| 457 | - | record.Route = record.Route.Clone() | |||
| 458 | - | return record, true, nil | |||
| 459 | - | } | |||
| 460 | - | } | |||
| 461 | - | return ports.BrokerHostRecord{}, false, nil | |||
| 462 | - | } | |||
| 463 | - | ||||
| 464 | - | func (r *Registry) AddHost(ctx context.Context, endpoint string, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |||
| 465 | - | if err := domain.ValidateRemoteHostTarget(endpoint); err != nil { | |||
| 466 | - | return domain.RemoteRegistration{}, err | |||
| 467 | - | } | |||
| 468 | - | if err := policy.Validate(); err != nil { | |||
| 469 | - | return domain.RemoteRegistration{}, err | |||
| 470 | - | } | |||
| 471 | - | r.mu.Lock() | |||
| 472 | - | defer r.mu.Unlock() | |||
| 473 | - | if err := r.mutableLocked(ctx); err != nil { | |||
| 474 | - | return domain.RemoteRegistration{}, err | |||
| 475 | - | } | |||
| 476 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 477 | - | for i := range records { | |||
| 478 | - | if records[i].Registration.Endpoint != endpoint { | |||
| 479 | - | continue | |||
| 480 | - | } | |||
| 481 | - | if records[i].Policy != policy { | |||
| 482 | - | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 483 | - | } | |||
| 484 | - | records[i].Pinned = true | |||
| 485 | - | if err := r.commitMembershipLocked(records); err != nil { | |||
| 486 | - | return domain.RemoteRegistration{}, err | |||
| 487 | - | } | |||
| 488 | - | return records[i].Registration, nil | |||
| 489 | - | } | |||
| 490 | - | id, err := r.newIncarnation() | |||
| 491 | - | if err != nil { | |||
| 492 | - | return domain.RemoteRegistration{}, err | |||
| 493 | - | } | |||
| 494 | - | reg, err := domain.NewRemoteRegistration(endpoint, id) | |||
| 495 | - | if err != nil { | |||
| 496 | - | return domain.RemoteRegistration{}, err | |||
| 497 | - | } | |||
| 498 | - | if reg.IsZero() { | |||
| 499 | - | return domain.RemoteRegistration{}, errors.New("broker: incarnation generator returned zero") | |||
| 500 | - | } | |||
| 501 | - | route, err := ports.BrokerRouteForTransport(policy.Transport, endpoint) | |||
| 502 | - | if err != nil { | |||
| 503 | - | return domain.RemoteRegistration{}, err | |||
| 504 | - | } | |||
| 505 | - | records = append(records, ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy, Route: route}) | |||
| 506 | - | if err := r.commitMembershipLocked(records); err != nil { | |||
| 507 | - | return domain.RemoteRegistration{}, err | |||
| 508 | - | } | |||
| 509 | - | return reg, nil | |||
| 510 | - | } | |||
| 511 | - | ||||
| 512 | - | // RemoveHost removes only the exact expected registration. | |||
| 513 | - | // BindAuthenticatedIdentity durably binds first-contact identity under the | |||
| 514 | - | // exact registration and policy fence. The same binding is idempotent; a | |||
| 515 | - | // different identity is refused and never overwrites authority. | |||
| 516 | - | func (r *Registry) BindAuthenticatedIdentity(ctx context.Context, request ports.BrokerIdentityBindingRequest) (ports.BrokerDaemonIdentity, error) { | |||
| 517 | - | if err := request.Validate(); err != nil { | |||
| 518 | - | return "", err | |||
| 519 | - | } | |||
| 520 | - | if request.Fence.Local { | |||
| 521 | - | return request.Identity, nil | |||
| 522 | - | } | |||
| 523 | - | if err := ctx.Err(); err != nil { | |||
| 524 | - | return "", err | |||
| 525 | - | } | |||
| 526 | - | r.mu.Lock() | |||
| 527 | - | defer r.mu.Unlock() | |||
| 528 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 529 | - | for index := range records { | |||
| 530 | - | record := &records[index] | |||
| 531 | - | if record.Identity == request.Identity && record.Policy != request.Policy { | |||
| 532 | - | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity is already bound under an incompatible policy")} | |||
| 533 | - | } | |||
| 534 | - | } | |||
| 535 | - | for index := range records { | |||
| 536 | - | record := &records[index] | |||
| 537 | - | if !record.Registration.Equal(request.Fence.Registration) || record.Policy != request.Policy { | |||
| 538 | - | continue | |||
| 539 | - | } | |||
| 540 | - | if record.Identity != "" { | |||
| 541 | - | if record.Identity != request.Identity { | |||
| 542 | - | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity conflicts with durable binding")} | |||
| 543 | - | } | |||
| 544 | - | return record.Identity, nil | |||
| 545 | - | } | |||
| 546 | - | record.Identity = request.Identity | |||
| 547 | - | if err := r.commitMembershipLocked(records); err != nil { | |||
| 548 | - | return "", err | |||
| 549 | - | } | |||
| 550 | - | return request.Identity, nil | |||
| 551 | - | } | |||
| 552 | - | return "", ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: errors.New("broker: stale identity binding fence")} | |||
| 553 | - | } | |||
| 554 | - | ||||
| 555 | - | func (r *Registry) RemoveHost(ctx context.Context, expected domain.RemoteRegistration) (bool, error) { | |||
| 556 | - | if err := expected.Validate(); err != nil { | |||
| 557 | - | return false, err | |||
| 558 | - | } | |||
| 559 | - | r.mu.Lock() | |||
| 560 | - | defer r.mu.Unlock() | |||
| 561 | - | if err := r.mutableLocked(ctx); err != nil { | |||
| 562 | - | return false, err | |||
| 563 | - | } | |||
| 564 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 565 | - | for i := range records { | |||
| 566 | - | if records[i].Registration.Endpoint != expected.Endpoint { | |||
| 567 | - | continue | |||
| 568 | - | } | |||
| 569 | - | if !records[i].Registration.Equal(expected) { | |||
| 570 | - | return false, ports.ErrBrokerHostConflict | |||
| 571 | - | } | |||
| 572 | - | records = append(records[:i], records[i+1:]...) | |||
| 573 | - | if err := r.commitMembershipLocked(records); err != nil { | |||
| 574 | - | return false, err | |||
| 575 | - | } | |||
| 576 | - | return true, nil | |||
| 577 | - | } | |||
| 578 | - | // A no-op still verifies the loaded revision. | |||
| 579 | - | return false, r.commitMembershipLocked(records) | |||
| 580 | - | } | |||
| 581 | - | ||||
| 582 | - | // UpdateHostPolicy updates exact authority and advances its generation. | |||
| 583 | - | func (r *Registry) UpdateHostPolicy(ctx context.Context, expected domain.RemoteRegistration, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |||
| 584 | - | if err := expected.Validate(); err != nil { | |||
| 585 | - | return domain.RemoteRegistration{}, err | |||
| 586 | - | } | |||
| 587 | - | if err := policy.Validate(); err != nil { | |||
| 588 | - | return domain.RemoteRegistration{}, err | |||
| 589 | - | } | |||
| 590 | - | r.mu.Lock() | |||
| 591 | - | defer r.mu.Unlock() | |||
| 592 | - | if err := r.mutableLocked(ctx); err != nil { | |||
| 593 | - | return domain.RemoteRegistration{}, err | |||
| 594 | - | } | |||
| 595 | - | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 596 | - | for i := range records { | |||
| 597 | - | if records[i].Registration.Endpoint != expected.Endpoint { | |||
| 598 | - | continue | |||
| 599 | - | } | |||
| 600 | - | if !records[i].Registration.Equal(expected) { | |||
| 601 | - | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 602 | - | } | |||
| 603 | - | if records[i].Policy == policy { | |||
| 604 | - | if err := r.commitMembershipLocked(records); err != nil { | |||
| 605 | - | return domain.RemoteRegistration{}, err | |||
| 606 | - | } | |||
| 607 | - | return expected, nil | |||
| 608 | - | } | |||
| 609 | - | if expected.Generation == ^domain.RemoteGeneration(0) { | |||
| 610 | - | return domain.RemoteRegistration{}, ports.ErrBrokerRevisionExhausted | |||
| 611 | - | } | |||
| 612 | - | route, err := ports.BrokerRouteForTransport(policy.Transport, expected.Endpoint) | |||
| 613 | - | if err != nil { | |||
| 614 | - | return domain.RemoteRegistration{}, err | |||
| 615 | - | } | |||
| 616 | - | records[i].Registration.Generation++ | |||
| 617 | - | records[i].Route = route | |||
| 618 | - | records[i].Policy = policy | |||
| 619 | - | if err := r.commitMembershipLocked(records); err != nil { | |||
| 620 | - | return domain.RemoteRegistration{}, err | |||
| 621 | - | } | |||
| 622 | - | return records[i].Registration, nil | |||
| 623 | - | } | |||
| 624 | - | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 625 | - | } | |||
| 626 | - | ||||
| 627 | - | func (r *Registry) mutableLocked(ctx context.Context) error { | |||
| 628 | - | if r.membershipMode != MembershipMutable { | |||
| 629 | - | return ports.ErrBrokerMembershipImmutable | |||
| 630 | - | } | |||
| 631 | - | if err := ctx.Err(); err != nil { | |||
| 632 | - | return err | |||
| 633 | - | } | |||
| 634 | - | if r.running.Load() == runStopped { | |||
| 635 | - | return ports.ErrBrokerRegistryClosed | |||
| 636 | - | } | |||
| 637 | - | if r.revisionExhaustedLocked() { | |||
| 638 | - | return ports.ErrBrokerRevisionExhausted | |||
| 639 | - | } | |||
| 640 | - | return nil | |||
| 641 | - | } | |||
| 642 | - | ||||
| 643 | - | // commitMembershipLocked is the sole mutable-membership durable commit path. | |||
| 644 | - | // It installs projection only after CAS success. | |||
| 645 | - | func (r *Registry) commitMembershipLocked(records []ports.BrokerHostRecord) error { | |||
| 646 | - | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 647 | - | return err | |||
| 648 | - | } | |||
| 649 | - | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | |||
| 650 | - | return errors.New("broker: remote hosts require a host probe") | |||
| 651 | - | } | |||
| 652 | - | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | |||
| 653 | - | order := make([]string, 0, len(records)) | |||
| 654 | - | for rank, record := range records { | |||
| 655 | - | observation := stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |||
| 656 | - | // A membership the registry cannot publish is refused here, before the | |||
| 657 | - | // store CAS, rather than persisted and left to freeze every later | |||
| 658 | - | // publication: an endpoint whose derived display hint is empty after the | |||
| 659 | - | // display rule drops its runes makes every snapshot carrying it invalid. | |||
| 660 | - | if err := observation.Validate(); err != nil { | |||
| 661 | - | return fmt.Errorf("broker: host record %q is not publishable: %w", record.Registration.Endpoint, err) | |||
| 662 | - | } | |||
| 663 | - | next[record.Registration.Endpoint] = observation | |||
| 664 | - | order = append(order, record.Registration.Endpoint) | |||
| 665 | - | } | |||
| 666 | - | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | |||
| 667 | - | return fmt.Errorf("broker: replace hosts: %w", err) | |||
| 668 | - | } | |||
| 669 | - | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: ports.CloneBrokerHostRecords(records)} | |||
| 670 | - | r.setHostsLocked(next, order) | |||
| 671 | - | return nil | |||
| 672 | - | } | |||
| 673 | - | ||||
| 674 | - | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |||
| 675 | - | // authority revision. It delegates to commitMembershipLocked, the same | |||
| 676 | - | // validated commit path the runtime mutations use, so membership whose | |||
| 677 | - | // projection could never be published is refused before the store CAS instead | |||
| 678 | - | // of being persisted and silently freezing every later publication. Success | |||
| 679 | - | // means membership is committed, not that advisory observations have flushed. | |||
| 680 | - | // Store errors (including stale authority) are returned without changing the | |||
| 681 | - | // projection, probe attempts, or local token. A settled run reports | |||
| 682 | - | // ports.ErrBrokerRegistryClosed and an exhausted revision series reports | |||
| 683 | - | // ports.ErrBrokerRevisionExhausted; both refuse the mutation before the store | |||
| 684 | - | // CAS, so a refusal never changes durable authority either. Conflicts are not | |||
| 685 | - | // retried or refreshed implicitly: reconcile by reopening the owner. Callers | |||
| 686 | - | // must supply fresh incarnations when removing and re-adding. Even identical | |||
| 687 | - | // records perform CAS so stale authority cannot report success. The synchronous | |||
| 688 | - | // write holds mu; Snapshot remains lock-free, but scheduling waits for | |||
| 689 | - | // membership durability. Observation writes remain asynchronous. | |||
| 690 | - | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |||
| 691 | - | var err error | |||
| 692 | - | records, err = ports.UpgradeBrokerHostRoutes(records) | |||
| 693 | - | if err != nil { | |||
| 694 | - | return err | |||
| 695 | - | } | |||
| 696 | - | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 697 | - | return err | |||
| 698 | - | } | |||
| 699 | - | r.mu.Lock() | |||
| 700 | - | defer r.mu.Unlock() | |||
| 701 | - | if r.running.Load() == runStopped { | |||
| 702 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |||
| 703 | - | } | |||
| 704 | - | if r.revisionExhaustedLocked() { | |||
| 705 | - | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |||
| 706 | - | } | |||
| 707 | - | return r.commitMembershipLocked(records) | |||
| 708 | - | } | |||
| 709 | - | ||||
| 710 | - | // setHostsLocked applies only a projection; the public path commits authority | |||
| 711 | - | // first. The caller holds mu through both steps, fencing probe completions. | |||
| 712 | - | // order is the registration order the projection was built from; it becomes | |||
| 713 | - | // the publication order of remote daemons. | |||
| 714 | - | func (r *Registry) setHostsLocked(next map[string]ports.BrokerDaemonObservation, order []string) { | |||
| 715 | - | if registrationsUnchanged(r.hosts, next, r.order, order) { | |||
| 716 | - | return | |||
| 717 | - | } | |||
| 718 | - | for endpoint, current := range r.hosts { | |||
| 719 | - | candidate, present := next[endpoint] | |||
| 720 | - | if present && current.Registration.Equal(candidate.Registration) && current.Policy == candidate.Policy { | |||
| 721 | - | // Unchanged authority: keep the live observation as is. Rank is the | |||
| 722 | - | // one authority field that follows membership order rather than the | |||
| 723 | - | // record, so it is re-stamped from the new projection. | |||
| 724 | - | kept := current | |||
| 725 | - | kept.Rank = candidate.Rank | |||
| 726 | - | next[endpoint] = kept | |||
| 727 | - | continue | |||
| 728 | - | } | |||
| 729 | - | // A retired registration owns no observation: cancelling its attempt | |||
| 730 | - | // releases a ctx-honoring probe immediately, and dropping its record | |||
| 731 | - | // means the retired completion can never clear or answer the | |||
| 732 | - | // replacement, and the replacement is never blocked behind it. The | |||
| 733 | - | // cancellation happens under the registry lock, so no replacement | |||
| 734 | - | // attempt can start before the retired context is cancelled. | |||
| 735 | - | r.cancelInflightLocked(endpoint) | |||
| 736 | - | delete(r.pending, endpoint) | |||
| 737 | - | if present { | |||
| 738 | - | // The endpoint stays live under a new registration, so its | |||
| 739 | - | // retirement is never published as a tombstone alongside a live | |||
| 740 | - | // host; the attempt token fences the stale completion. | |||
| 741 | - | delete(r.tombstones, endpoint) | |||
| 742 | - | continue | |||
| 743 | - | } | |||
| 744 | - | r.retireLocked(current.Registration) | |||
| 745 | - | } | |||
| 746 | - | for endpoint := range next { | |||
| 747 | - | // Re-adding an endpoint supersedes its tombstone: a live host and a | |||
| 748 | - | // tombstone for the same endpoint are mutually exclusive. | |||
| 749 | - | delete(r.tombstones, endpoint) | |||
| 750 | - | } | |||
| 751 | - | r.hosts = next | |||
| 752 | - | r.order = order | |||
| 753 | - | r.publishLocked(true) | |||
| 754 | - | r.hint() | |||
| 755 | - | } | |||
| 756 | - | ||||
| 757 | - | // cancelInflightLocked cancels the in-flight attempt for one endpoint and | |||
| 758 | - | // drops its record. Callers must hold r.mu: the retired attempt's context is | |||
| 759 | - | // cancelled before the record disappears, so no replacement attempt can start | |||
| 760 | - | // while the retired probe's context is still live. | |||
| 761 | - | func (r *Registry) cancelInflightLocked(endpoint string) { | |||
| 762 | - | if attempt := r.inflight[endpoint]; attempt != nil { | |||
| 763 | - | attempt.cancel() | |||
| 764 | - | delete(r.inflight, endpoint) | |||
| 765 | - | } | |||
| 766 | - | } | |||
| 767 | - | ||||
| 768 | - | // cancelAllInflightLocked cancels every in-flight attempt and drops the | |||
| 769 | - | // records. Callers must hold r.mu. | |||
| 770 | - | func (r *Registry) cancelAllInflightLocked() { | |||
| 771 | - | for endpoint, attempt := range r.inflight { | |||
| 772 | - | attempt.cancel() | |||
| 773 | - | delete(r.inflight, endpoint) | |||
| 774 | - | } | |||
| 775 | - | } | |||
| 776 | - | ||||
| 777 | - | // RequestProbe is non-blocking and coalesces concurrent demand per endpoint. | |||
| 778 | - | // A disabled registry never observes, so the demand is dropped without | |||
| 779 | - | // scheduling anything. An empty endpoint requests a local re-probe instead of a | |||
| 780 | - | // configured remote: the broker's own machine daemon is re-observed ahead of | |||
| 781 | - | // its scheduled freshness window, honored by dispatchLocalLocked exactly like | |||
| 782 | - | // pending honors remote demand. A registry with no local observation producer | |||
| 783 | - | // drops it, just like an unknown remote endpoint is dropped. | |||
| 784 | - | func (r *Registry) RequestProbe(endpoint string) { | |||
| 785 | - | if r.observationDisabled { | |||
| 786 | - | return | |||
| 787 | - | } | |||
| 788 | - | r.mu.Lock() | |||
| 789 | - | if endpoint == "" { | |||
| 790 | - | if r.local != nil { | |||
| 791 | - | r.localPending = true | |||
| 792 | - | } | |||
| 793 | - | } else if _, ok := r.hosts[endpoint]; ok { | |||
| 794 | - | r.pending[endpoint] = true | |||
| 795 | - | } | |||
| 796 | - | r.mu.Unlock() | |||
| 797 | - | r.hint() | |||
| 798 | - | } | |||
| 799 | - | ||||
| 800 | - | // SetDemand records one live client subscription opening (active=true) or | |||
| 801 | - | // closing (active=false). internal/usecase/broker/service.go calls it from | |||
| 802 | - | // Service.Subscribe and serviceSubscription.Close, so the demand count tracks | |||
| 803 | - | // exactly the connections a client is actively watching. The first | |||
| 804 | - | // subscription requests one immediate observation per host; while demand | |||
| 805 | - | // stays positive, the demand cadence schedules later observations. With no | |||
| 806 | - | // subscription, only explicit RequestProbe calls dispatch. A disabled registry | |||
| 807 | - | // still counts subscriptions but never observes. | |||
| 808 | - | func (r *Registry) SetDemand(active bool) { | |||
| 809 | - | r.mu.Lock() | |||
| 810 | - | if active { | |||
| 811 | - | if r.demand == 0 { | |||
| 812 | - | r.localPending = r.local != nil | |||
| 813 | - | for endpoint := range r.hosts { | |||
| 814 | - | r.pending[endpoint] = true | |||
| 815 | - | } | |||
| 816 | - | } | |||
| 817 | - | r.demand++ | |||
| 818 | - | } else if r.demand > 0 { | |||
| 819 | - | r.demand-- | |||
| 820 | - | } | |||
| 821 | - | r.mu.Unlock() | |||
| 822 | - | r.hint() | |||
| 823 | - | } | |||
| 824 | - | ||||
| 825 | - | // freshForLocked returns the demand cadence while subscribed. An absent | |||
| 826 | - | // daemon uses the longer freshFor interval instead. Callers must hold r.mu. | |||
| 827 | - | func (r *Registry) freshForLocked(local bool) time.Duration { | |||
| 828 | - | if r.demand <= 0 { | |||
| 829 | - | return r.freshFor | |||
| 830 | - | } | |||
| 831 | - | if local { | |||
| 832 | - | return r.demandFreshForLocal | |||
| 833 | - | } | |||
| 834 | - | return r.demandFreshForRemote | |||
| 835 | - | } | |||
| 836 | - | ||||
| 837 | - | func (r *Registry) hint() { | |||
| 838 | - | select { | |||
| 839 | - | case r.wake <- struct{}{}: | |||
| 840 | - | default: | |||
| 841 | - | } | |||
| 842 | - | } | |||
| 843 | - | ||||
| 844 | - | // Run schedules observations until cancellation. Run owns the registry | |||
| 845 | - | // lifecycle and may execute exactly once: a second call is refused so a | |||
| 846 | - | // concurrent or restarted caller can never duplicate scheduling. On return it | |||
| 847 | - | // clears in-flight observation state, publishes the settled projection, and | |||
| 848 | - | // flushes durable persistence before any further write is refused. | |||
| 849 | - | // | |||
| 850 | - | // An observation-disabled registry runs the same lifecycle and drain but never | |||
| 851 | - | // schedules observation: it arms no timer, starts no probe, and drops every | |||
| 852 | - | // reconcile. It still owns and drains its durable writer, so a shutdown never | |||
| 853 | - | // returns before the newest staged publication has reached the store. | |||
| 854 | - | func (r *Registry) Run(ctx context.Context) { | |||
| 855 | - | if !r.running.CompareAndSwap(runIdle, runActive) { | |||
| 856 | - | r.log.Warn("broker: registry Run refused; single-run already started or finished") | |||
| 857 | - | return | |||
| 858 | - | } | |||
| 859 | - | defer r.settle() | |||
| 860 | - | if r.observationDisabled { | |||
| 861 | - | <-ctx.Done() | |||
| 862 | - | return | |||
| 863 | - | } | |||
| 864 | - | timer := r.clock.NewTimer(0) | |||
| 865 | - | defer timer.Stop() | |||
| 866 | - | results := make(chan probeResult, ports.BrokerMaxHosts) | |||
| 867 | - | for { | |||
| 868 | - | now := r.clock.Now() | |||
| 869 | - | r.dispatch(ctx, now, results) | |||
| 870 | - | timer.Reset(r.nextDelay(now)) | |||
| 871 | - | select { | |||
| 872 | - | case <-ctx.Done(): | |||
| 873 | - | return | |||
| 874 | - | case <-r.wake: | |||
| 875 | - | case <-timer.C(): | |||
| 876 | - | case result := <-results: | |||
| 877 | - | if ctx.Err() != nil { | |||
| 878 | - | // Shutdown wins: a cancellation is never recorded as an | |||
| 879 | - | // observation failure. | |||
| 880 | - | return | |||
| 881 | - | } | |||
| 882 | - | r.apply(result) | |||
| 883 | - | } | |||
| 884 | - | } | |||
| 885 | - | } | |||
| 886 | - | ||||
| 887 | - | // settle is Run's cleanup: nothing stays in flight, the published projection | |||
| 888 | - | // carries no transient Checking flag, durable persistence is flushed, and | |||
| 889 | - | // every later Run call is refused. | |||
| 890 | - | func (r *Registry) settle() { | |||
| 891 | - | r.mu.Lock() | |||
| 892 | - | r.cancelAllInflightLocked() | |||
| 893 | - | if r.localAttempt != nil { | |||
| 894 | - | r.localAttempt.cancel() | |||
| 895 | - | r.localAttempt = nil | |||
| 896 | - | } | |||
| 897 | - | cleared := false | |||
| 898 | - | if r.local != nil && r.localHost.Checking { | |||
| 899 | - | r.localHost.Checking = false | |||
| 900 | - | cleared = true | |||
| 901 | - | } | |||
| 902 | - | for endpoint, host := range r.hosts { | |||
| 903 | - | if !host.Checking { | |||
| 904 | - | continue | |||
| 905 | - | } | |||
| 906 | - | host.Checking = false | |||
| 907 | - | r.hosts[endpoint] = host | |||
| 908 | - | cleared = true | |||
| 909 | - | } | |||
| 910 | - | if cleared { | |||
| 911 | - | r.publishLocked(true) | |||
| 912 | - | } | |||
| 913 | - | r.running.Store(runStopped) | |||
| 914 | - | r.mu.Unlock() | |||
| 915 | - | r.store.close() | |||
| 916 | - | } | |||
| 917 | - | ||||
| 918 | - | // probeAttempt is one admitted observation: the registry-wide token that | |||
| 919 | - | // fences its completion, and the cancel for the context derived from the run | |||
| 920 | - | // context. Cancelling the derived context retires the attempt: a probe that | |||
| 921 | - | // honors cancellation returns promptly, so a replaced or removed endpoint | |||
| 922 | - | // never accumulates overlapping probes, and settle releases every attempt | |||
| 923 | - | // before the run loop exits. | |||
| 924 | - | type probeAttempt struct { | |||
| 925 | - | token uint64 | |||
| 926 | - | cancel context.CancelFunc | |||
| 927 | - | } | |||
| 928 | - | ||||
| 929 | - | // probeResult is one completed observation bound to the exact attempt token | |||
| 930 | - | // that started it. | |||
| 931 | - | type probeResult struct { | |||
| 932 | - | endpoint string | |||
| 933 | - | registration domain.RemoteRegistration | |||
| 934 | - | attempt uint64 | |||
| 935 | - | snapshot ports.BrokerDaemonObservation | |||
| 936 | - | err error | |||
| 937 | - | at time.Time | |||
| 938 | - | // local marks the result of one local probe attempt, which carries no | |||
| 939 | - | // endpoint or registration and is applied to the configured local entry. | |||
| 940 | - | local bool | |||
| 941 | - | } | |||
| 942 | - | ||||
| 943 | - | func (r *Registry) dispatch(ctx context.Context, now time.Time, results chan<- probeResult) { | |||
| 944 | - | r.mu.Lock() | |||
| 945 | - | if r.demand == 0 && len(r.pending) == 0 && !r.localPending { | |||
| 946 | - | r.mu.Unlock() | |||
| 947 | - | return | |||
| 948 | - | } | |||
| 949 | - | started := false | |||
| 950 | - | for endpoint, host := range r.hosts { | |||
| 951 | - | if _, inflight := r.inflight[endpoint]; inflight { | |||
| 952 | - | continue | |||
| 953 | - | } | |||
| 954 | - | explicit := r.pending[endpoint] | |||
| 955 | - | if !explicit && (r.demand == 0 || !host.NextDue.IsZero() && now.Before(host.NextDue)) { | |||
| 956 | - | continue | |||
| 957 | - | } | |||
| 958 | - | delete(r.pending, endpoint) | |||
| 959 | - | r.attempts++ | |||
| 960 | - | attempt := r.attempts | |||
| 961 | - | probeCtx, cancel := context.WithCancel(ctx) | |||
| 962 | - | r.inflight[endpoint] = &probeAttempt{token: attempt, cancel: cancel} | |||
| 963 | - | if visibleCheck(host, explicit) { | |||
| 964 | - | started = true | |||
| 965 | - | host.Checking = true | |||
| 966 | - | } | |||
| 967 | - | host.LastAttempt = now | |||
| 968 | - | r.hosts[endpoint] = host | |||
| 969 | - | registration := host.Registration | |||
| 970 | - | // The deadline is armed here, under r.mu and at dispatch time, so the | |||
| 971 | - | // attempt's bound starts with the attempt rather than whenever its | |||
| 972 | - | // goroutine is scheduled. | |||
| 973 | - | deadline := r.clock.NewTimer(r.probeTimeout) | |||
| 974 | - | go func(endpoint string, registration domain.RemoteRegistration, attempt uint64, probeCtx context.Context) { | |||
| 975 | - | snapshot, err := observeBounded(probeCtx, deadline, func(ctx context.Context) (ports.BrokerDaemonObservation, error) { | |||
| 976 | - | return r.probe.Probe(ctx, registration) | |||
| 977 | - | }) | |||
| 978 | - | result := probeResult{endpoint: endpoint, registration: registration, attempt: attempt, snapshot: snapshot, err: err, at: r.clock.Now()} | |||
| 979 | - | select { | |||
| 980 | - | case results <- result: | |||
| 981 | - | case <-probeCtx.Done(): | |||
| 982 | - | // The attempt was retired (replaced, removed, or shutdown), so | |||
| 983 | - | // its completion is already fenced: dropping it keeps churn from | |||
| 984 | - | // queueing results the run loop would only discard. | |||
| 985 | - | } | |||
| 986 | - | }(endpoint, registration, attempt, probeCtx) | |||
| 987 | - | } | |||
| 988 | - | // Only an observation that visibly started changes the projection; an | |||
| 989 | - | // idle dispatch or a routine refresh must not bump the revision or wake | |||
| 990 | - | // subscribers. The in-flight projection is never persisted: Checking is | |||
| 991 | - | // transient. | |||
| 992 | - | if r.dispatchLocalLocked(ctx, now, results) { | |||
| 993 | - | started = true | |||
| 994 | - | } | |||
| 995 | - | if started { | |||
| 996 | - | r.publishLocked(false) | |||
| 997 | - | } | |||
| 998 | - | r.mu.Unlock() | |||
| 999 | - | } | |||
| 1000 | - | ||||
| 1001 | - | // visibleCheck reports whether starting an observation publishes the transient | |||
| 1002 | - | // Checking state. A routine scheduled refresh of a reachable daemon stays | |||
| 1003 | - | // silent until its result lands, so an idle watched broker publishes once per | |||
| 1004 | - | // observation instead of twice; an explicit request or a daemon that is not | |||
| 1005 | - | // confirmed reachable still shows that it is being checked. | |||
| 1006 | - | func visibleCheck(host ports.BrokerDaemonObservation, explicit bool) bool { | |||
| 1007 | - | return explicit || host.Availability != domain.RemoteAvailabilityReachable | |||
| 1008 | - | } | |||
| 1009 | - | ||||
| 1010 | - | // observeBounded runs one observation under its attempt deadline. retire is the | |||
| 1011 | - | // attempt's retirement context: a replaced, removed, or settled attempt returns | |||
| 1012 | - | // at once and its completion is fenced by the caller. The probe itself runs on | |||
| 1013 | - | // a context derived from retire that the deadline also cancels, so a | |||
| 1014 | - | // ctx-honoring probe unwinds its dial, handshake, or exchange when either | |||
| 1015 | - | // fires. The attempt never waits for the probe past its deadline: a probe that | |||
| 1016 | - | // ignores cancellation is abandoned (its late result is discarded) and the | |||
| 1017 | - | // attempt still reports errProbeTimeout, so the endpoint's single in-flight | |||
| 1018 | - | // slot is always released. | |||
| 1019 | - | func observeBounded(retire context.Context, deadline ports.Timer, observe func(context.Context) (ports.BrokerDaemonObservation, error)) (ports.BrokerDaemonObservation, error) { | |||
| 1020 | - | defer deadline.Stop() | |||
| 1021 | - | probeCtx, cancel := context.WithCancelCause(retire) | |||
| 1022 | - | defer cancel(nil) | |||
| 1023 | - | type outcome struct { | |||
| 1024 | - | snapshot ports.BrokerDaemonObservation | |||
| 1025 | - | err error | |||
| 1026 | - | } | |||
| 1027 | - | done := make(chan outcome, 1) | |||
| 1028 | - | go func() { | |||
| 1029 | - | snapshot, err := observe(probeCtx) | |||
| 1030 | - | done <- outcome{snapshot: snapshot, err: err} | |||
| 1031 | - | }() | |||
| 1032 | - | select { | |||
| 1033 | - | case result := <-done: | |||
| 1034 | - | return result.snapshot, result.err | |||
| 1035 | - | case <-deadline.C(): | |||
| 1036 | - | cancel(errProbeTimeout) | |||
| 1037 | - | return ports.BrokerDaemonObservation{}, errProbeTimeout | |||
| 1038 | - | case <-retire.Done(): | |||
| 1039 | - | return ports.BrokerDaemonObservation{}, retire.Err() | |||
| 1040 | - | } | |||
| 1041 | - | } | |||
| 1042 | - | ||||
| 1043 | - | func (r *Registry) apply(result probeResult) { | |||
| 1044 | - | r.mu.Lock() | |||
| 1045 | - | defer r.mu.Unlock() | |||
| 1046 | - | if result.local { | |||
| 1047 | - | r.applyLocal(result) | |||
| 1048 | - | return | |||
| 1049 | - | } | |||
| 1050 | - | current, ok := r.hosts[result.endpoint] | |||
| 1051 | - | attempt := r.inflight[result.endpoint] | |||
| 1052 | - | if !ok || !current.Registration.Equal(result.registration) || attempt == nil || attempt.token != result.attempt { | |||
| 1053 | - | // A stale completion touches neither current authority nor the | |||
| 1054 | - | // in-flight slot, which may already belong to a newer registration or | |||
| 1055 | - | // attempt. | |||
| 1056 | - | return | |||
| 1057 | - | } | |||
| 1058 | - | delete(r.inflight, result.endpoint) | |||
| 1059 | - | // The probe has returned, so releasing its derived context here only | |||
| 1060 | - | // frees the attempt's resources; the record is gone before any new | |||
| 1061 | - | // attempt can start. | |||
| 1062 | - | attempt.cancel() | |||
| 1063 | - | current.Checking = false | |||
| 1064 | - | current.LastAttempt = result.at | |||
| 1065 | - | ||||
| 1066 | - | kind, availability, failed := observationOutcome(result) | |||
| 1067 | - | if !failed && availability == domain.RemoteAvailabilityNoDaemon { | |||
| 1068 | - | current.Availability = availability | |||
| 1069 | - | current.LastSuccess = result.at | |||
| 1070 | - | // An absent daemon is stable: demand does not require a fresh SSH | |||
| 1071 | - | // handshake every two seconds just to confirm it is still absent. | |||
| 1072 | - | current.NextDue = result.at.Add(r.jitter(r.freshFor, result.endpoint, result.attempt)) | |||
| 1073 | - | current.ConsecutiveFailures = 0 | |||
| 1074 | - | current.LastFailure = domain.RemoteFailure{} | |||
| 1075 | - | current.Identity = "" | |||
| 1076 | - | current.Incarnation = ports.BrokerDaemonIncarnation{} | |||
| 1077 | - | current.ProtocolVersion = 0 | |||
| 1078 | - | current.Capabilities = 0 | |||
| 1079 | - | current.InventoryKnown = false | |||
| 1080 | - | current.Sessions = nil | |||
| 1081 | - | r.hosts[result.endpoint] = current | |||
| 1082 | - | r.publishLocked(true) | |||
| 1083 | - | return | |||
| 1084 | - | } | |||
| 1085 | - | if !failed { | |||
| 1086 | - | // Configured authority comes only from the registry's own projection: | |||
| 1087 | - | // a probe result supplies observed state, never endpoint, registration, | |||
| 1088 | - | // policy, display origin, or rank. | |||
| 1089 | - | observed := stampObservationAuthority(result.snapshot.Clone(), current) | |||
| 1090 | - | observed.Checking = false | |||
| 1091 | - | observed.LastAttempt = result.at | |||
| 1092 | - | observed.LastSuccess = result.at | |||
| 1093 | - | observed.NextDue = result.at.Add(r.jitter(r.freshForLocked(false), result.endpoint, result.attempt)) | |||
| 1094 | - | observed.ConsecutiveFailures = 0 | |||
| 1095 | - | observed.LastFailure = domain.RemoteFailure{} | |||
| 1096 | - | observed.FailureEpisode = current.FailureEpisode | |||
| 1097 | - | // The published observation must be fully valid once authority is | |||
| 1098 | - | // stamped: a probe that invents partial identity (identity without a | |||
| 1099 | - | // protocol version) or an inventory without a known catalogue is an | |||
| 1100 | - | // invalid response, never a published observation. The durable rule is | |||
| 1101 | - | // re-checked too: a projection the offline store would reject is never | |||
| 1102 | - | // published as if it had persisted. The store applies the same rule. | |||
| 1103 | - | if err := observed.Validate(); err != nil { | |||
| 1104 | - | r.log.Debug("broker: observation is invalid", "endpoint", result.endpoint, "err", err) | |||
| 1105 | - | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |||
| 1106 | - | } else if err := ports.ValidateDurableHostProjection(observed); err != nil { | |||
| 1107 | - | r.log.Debug("broker: observation is not persistable", "endpoint", result.endpoint, "err", err) | |||
| 1108 | - | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |||
| 1109 | - | } else { | |||
| 1110 | - | r.hosts[result.endpoint] = observed | |||
| 1111 | - | r.publishLocked(true) | |||
| 1112 | - | return | |||
| 1113 | - | } | |||
| 1114 | - | } | |||
| 1115 | - | // Every non-reachable outcome is a typed failure on the capped retry | |||
| 1116 | - | // cadence: the last success, last-known inventory, and failure episode | |||
| 1117 | - | // counter stay authoritative. | |||
| 1118 | - | current.Availability = availability | |||
| 1119 | - | if current.ConsecutiveFailures == 0 { | |||
| 1120 | - | current.FailureEpisode++ | |||
| 1121 | - | } | |||
| 1122 | - | current.ConsecutiveFailures++ | |||
| 1123 | - | current.LastFailure = domain.RemoteFailure{Kind: kind, Err: result.err} | |||
| 1124 | - | delay := r.retryDelay(current.ConsecutiveFailures) | |||
| 1125 | - | if r.demand > 0 { | |||
| 1126 | - | delay = demandRetryDelay(r.demandFreshForRemote, current.ConsecutiveFailures) | |||
| 1127 | - | } | |||
| 1128 | - | current.NextDue = result.at.Add(r.jitter(delay, result.endpoint, result.attempt)) | |||
| 1129 | - | r.log.Debug("broker_probe_failed", "endpoint", result.endpoint, "kind", kind, "failures", current.ConsecutiveFailures, "err", result.err, "cause", errors.Unwrap(result.err)) | |||
| 1130 | - | r.hosts[result.endpoint] = current | |||
| 1131 | - | r.publishLocked(true) | |||
| 1132 | - | } | |||
| 1133 | - | ||||
| 1134 | - | // observationOutcome classifies a completed observation. It reports | |||
| 1135 | - | // failed=false only for a confirmed reachable result; every other outcome | |||
| 1136 | - | // carries the typed failure kind and the availability to publish. The caller | |||
| 1137 | - | // additionally checks the adopted projection against the durable rules | |||
| 1138 | - | // (ports.ValidateDurableHostProjection), which cover the session bound and | |||
| 1139 | - | // catalogue validity for the exact incarnation and success time. | |||
| 1140 | - | func observationOutcome(result probeResult) (domain.RemoteFailureKind, domain.RemoteAvailability, bool) { | |||
| 1141 | - | if result.err != nil { | |||
| 1142 | - | kind := domain.RemoteFailureTransport | |||
| 1143 | - | var typed domain.RemoteFailure | |||
| 1144 | - | var brokerErr ports.BrokerError | |||
| 1145 | - | switch { | |||
| 1146 | - | case errors.As(result.err, &typed) && typed.Kind != domain.RemoteFailureNone: | |||
| 1147 | - | // The dialer classified the failure (for example SSH authentication). | |||
| 1148 | - | kind = typed.Kind | |||
| 1149 | - | case errors.As(result.err, &brokerErr) && (brokerErr.Code == ports.BrokerErrorConflictingPolicy || brokerErr.Code == ports.BrokerErrorIncompatible): | |||
| 1150 | - | // SSH worked; the daemon's identity or version was refused. | |||
| 1151 | - | kind = domain.RemoteFailureIncompatible | |||
| 1152 | - | case errors.Is(result.err, context.DeadlineExceeded): | |||
| 1153 | - | kind = domain.RemoteFailureTimeout | |||
| 1154 | - | } | |||
| 1155 | - | return kind, availabilityFor(kind), true | |||
| 1156 | - | } | |||
| 1157 | - | switch result.snapshot.Availability { | |||
| 1158 | - | case domain.RemoteAvailabilityReachable: | |||
| 1159 | - | return domain.RemoteFailureNone, domain.RemoteAvailabilityReachable, false | |||
| 1160 | - | case domain.RemoteAvailabilityNoDaemon: | |||
| 1161 | - | return domain.RemoteFailureNone, domain.RemoteAvailabilityNoDaemon, false | |||
| 1162 | - | case domain.RemoteAvailabilityIncompatible: | |||
| 1163 | - | return domain.RemoteFailureIncompatible, domain.RemoteAvailabilityIncompatible, true | |||
| 1164 | - | case domain.RemoteAvailabilityAuthFailed: | |||
| 1165 | - | return domain.RemoteFailureAuthentication, domain.RemoteAvailabilityAuthFailed, true | |||
| 1166 | - | case domain.RemoteAvailabilityInvalidResponse: | |||
| 1167 | - | return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |||
| 1168 | - | default: | |||
| 1169 | - | // Unknown or Unreachable without an error still means reachability was | |||
| 1170 | - | // not confirmed, so it retries on the typed backoff cadence. | |||
| 1171 | - | return domain.RemoteFailureTransport, domain.RemoteAvailabilityUnreachable, true | |||
| 1172 | - | } | |||
| 1173 | - | } | |||
| 1174 | - | ||||
| 1175 | - | // availabilityFor maps a typed failure kind to the published availability. | |||
| 1176 | - | // The typed kind is preserved separately for notice policy. | |||
| 1177 | - | func availabilityFor(kind domain.RemoteFailureKind) domain.RemoteAvailability { | |||
| 1178 | - | switch kind { | |||
| 1179 | - | case domain.RemoteFailureAuthentication: | |||
| 1180 | - | return domain.RemoteAvailabilityAuthFailed | |||
| 1181 | - | case domain.RemoteFailureIncompatible: | |||
| 1182 | - | return domain.RemoteAvailabilityIncompatible | |||
| 1183 | - | case domain.RemoteFailureInvalidResponse: | |||
| 1184 | - | return domain.RemoteAvailabilityInvalidResponse | |||
| 1185 | - | default: | |||
| 1186 | - | return domain.RemoteAvailabilityUnreachable | |||
| 1187 | - | } | |||
| 1188 | - | } | |||
| 1189 | - | ||||
| 1190 | - | // demandRetryDelay is the watched-host failure backoff: the demand cadence | |||
| 1191 | - | // doubled per consecutive failure, capped at demandRetryMax. | |||
| 1192 | - | func demandRetryDelay(base time.Duration, failures uint) time.Duration { | |||
| 1193 | - | delay := base | |||
| 1194 | - | for i := uint(1); i < failures && delay < demandRetryMax; i++ { | |||
| 1195 | - | delay *= 2 | |||
| 1196 | - | } | |||
| 1197 | - | return min(delay, max(base, demandRetryMax)) | |||
| 1198 | - | } | |||
| 1199 | - | ||||
| 1200 | - | // retryDelay is the capped exponential retry cadence for a failure streak. | |||
| 1201 | - | func (r *Registry) retryDelay(failures uint) time.Duration { | |||
| 1202 | - | delay := r.retryBase | |||
| 1203 | - | for i := uint(1); i < failures; i++ { | |||
| 1204 | - | if delay >= r.retryMax { | |||
| 1205 | - | return r.retryMax | |||
| 1206 | - | } | |||
| 1207 | - | delay *= 2 | |||
| 1208 | - | } | |||
| 1209 | - | if delay > r.retryMax { | |||
| 1210 | - | return r.retryMax | |||
| 1211 | - | } | |||
| 1212 | - | return delay | |||
| 1213 | - | } | |||
| 1214 | - | ||||
| 1215 | - | // jittered spreads a base interval deterministically across endpoint and | |||
| 1216 | - | // attempt: identical inputs always yield the identical duration, and the | |||
| 1217 | - | // result stays within 90%-110% of the base. No wall clock or global | |||
| 1218 | - | // randomness participates, so retry and healthy-refresh schedules are | |||
| 1219 | - | // reproducible instead of racing a shared PRNG. | |||
| 1220 | - | func jittered(base time.Duration, endpoint string, attempt uint64) time.Duration { | |||
| 1221 | - | if base <= 0 { | |||
| 1222 | - | return base | |||
| 1223 | - | } | |||
| 1224 | - | h := fnv.New32a() | |||
| 1225 | - | h.Write([]byte(endpoint)) | |||
| 1226 | - | var counter [8]byte | |||
| 1227 | - | binary.BigEndian.PutUint64(counter[:], attempt) | |||
| 1228 | - | h.Write(counter[:]) | |||
| 1229 | - | spread := int64(h.Sum32() % 21) | |||
| 1230 | - | return base*90/100 + time.Duration(spread)*base/100 | |||
| 1231 | - | } | |||
| 1232 | - | ||||
| 1233 | - | func (r *Registry) nextDelay(now time.Time) time.Duration { | |||
| 1234 | - | r.mu.Lock() | |||
| 1235 | - | defer r.mu.Unlock() | |||
| 1236 | - | if r.demand == 0 { | |||
| 1237 | - | if r.localPending || len(r.pending) > 0 { | |||
| 1238 | - | return 0 | |||
| 1239 | - | } | |||
| 1240 | - | return time.Hour | |||
| 1241 | - | } | |||
| 1242 | - | var earliest time.Time | |||
| 1243 | - | if r.local != nil && r.localAttempt == nil { | |||
| 1244 | - | if r.localHost.NextDue.IsZero() { | |||
| 1245 | - | return 0 | |||
| 1246 | - | } | |||
| 1247 | - | if earliest.IsZero() || r.localHost.NextDue.Before(earliest) { | |||
| 1248 | - | earliest = r.localHost.NextDue | |||
| 1249 | - | } | |||
| 1250 | - | } | |||
| 1251 | - | for endpoint, host := range r.hosts { | |||
| 1252 | - | // An observation already in flight is completed by its result, never | |||
| 1253 | - | // by the schedule; waiting on it here would spin the run loop. | |||
| 1254 | - | if _, inflight := r.inflight[endpoint]; inflight { | |||
| 1255 | - | continue | |||
| 1256 | - | } | |||
| 1257 | - | if r.pending[endpoint] || host.NextDue.IsZero() { | |||
| 1258 | - | return 0 | |||
| 1259 | - | } | |||
| 1260 | - | if earliest.IsZero() || host.NextDue.Before(earliest) { | |||
| 1261 | - | earliest = host.NextDue | |||
| 1262 | - | } | |||
| 1263 | - | } | |||
| 1264 | - | if earliest.IsZero() { | |||
| 1265 | - | return time.Hour | |||
| 1266 | - | } | |||
| 1267 | - | if delay := earliest.Sub(now); delay > 0 { | |||
| 1268 | - | return delay | |||
| 1269 | - | } | |||
| 1270 | - | return 0 | |||
| 1271 | - | } | |||
| 1272 | - | ||||
| 1273 | - | // registrationsUnchanged reports exact membership equality: the same endpoints | |||
| 1274 | - | // in the same registration order, bound to the same registration identities and | |||
| 1275 | - | // policy. Publication order and presentation rank are functions of durable | |||
| 1276 | - | // membership order, so a reordering of identical records is a membership change | |||
| 1277 | - | // that must republish; otherwise the same durable state yields different | |||
| 1278 | - | // publications depending on process history. Projection fields such as | |||
| 1279 | - | // availability, checking, or inventory never participate. | |||
| 1280 | - | func registrationsUnchanged(current, next map[string]ports.BrokerDaemonObservation, currentOrder, nextOrder []string) bool { | |||
| 1281 | - | if len(current) != len(next) { | |||
| 1282 | - | return false | |||
| 1283 | - | } | |||
| 1284 | - | if !slices.Equal(currentOrder, nextOrder) { | |||
| 1285 | - | return false | |||
| 1286 | - | } | |||
| 1287 | - | for endpoint, candidate := range next { | |||
| 1288 | - | existing, ok := current[endpoint] | |||
| 1289 | - | if !ok || !existing.Registration.Equal(candidate.Registration) || existing.Policy != candidate.Policy { | |||
| 1290 | - | return false | |||
| 1291 | - | } | |||
| 1292 | - | } | |||
| 1293 | - | return true | |||
| 1294 | - | } | |||
| 1295 | - | ||||
| 1296 | - | // retireLocked records the tombstone for one removed registration at the | |||
| 1297 | - | // revision of the publication that carries it. Mutation paths refuse an | |||
| 1298 | - | // exhausted series before retiring, so the retirement revision can never wrap | |||
| 1299 | - | // to zero or fall below the last published one. | |||
| 1300 | - | func (r *Registry) retireLocked(registration domain.RemoteRegistration) { | |||
| 1301 | - | retired := r.revision + 1 | |||
| 1302 | - | r.tombstones[registration.Endpoint] = ports.BrokerHostTombstone{ | |||
| 1303 | - | Endpoint: registration.Endpoint, | |||
| 1304 | - | Registration: registration, | |||
| 1305 | - | RetiredRevision: retired, | |||
| 1306 | - | } | |||
| 1307 | - | r.pruneTombstonesLocked() | |||
| 1308 | - | } | |||
| 1309 | - | ||||
| 1310 | - | // revisionExhaustedLocked reports whether this epoch has no revision left to | |||
| 1311 | - | // publish, latching the exhausted state on first detection. Callers must hold | |||
| 1312 | - | // r.mu. | |||
| 1313 | - | func (r *Registry) revisionExhaustedLocked() bool { | |||
| 1314 | - | if r.revisionExhausted { | |||
| 1315 | - | return true | |||
| 1316 | - | } | |||
| 1317 | - | if r.revision != ^ports.BrokerRevision(0) { | |||
| 1318 | - | return false | |||
| 1319 | - | } | |||
| 1320 | - | r.revisionExhausted = true | |||
| 1321 | - | r.log.Error("broker: revision series exhausted; refusing further publication", "epoch", r.epoch) | |||
| 1322 | - | return true | |||
| 1323 | - | } | |||
| 1324 | - | ||||
| 1325 | - | // pruneTombstonesLocked keeps only the newest BrokerMaxTombstones retirements | |||
| 1326 | - | // so an endpoint that churns can never grow the published set without bound. | |||
| 1327 | - | func (r *Registry) pruneTombstonesLocked() { | |||
| 1328 | - | for len(r.tombstones) > ports.BrokerMaxTombstones { | |||
| 1329 | - | oldest := "" | |||
| 1330 | - | var oldestRevision ports.BrokerRevision | |||
| 1331 | - | for endpoint, tombstone := range r.tombstones { | |||
| 1332 | - | if oldest == "" || tombstone.RetiredRevision < oldestRevision || | |||
| 1333 | - | (tombstone.RetiredRevision == oldestRevision && endpoint < oldest) { | |||
| 1334 | - | oldest, oldestRevision = endpoint, tombstone.RetiredRevision | |||
| 1335 | - | } | |||
| 1336 | - | } | |||
| 1337 | - | delete(r.tombstones, oldest) | |||
| 1338 | - | } | |||
| 1339 | - | } | |||
| 1340 | - | ||||
| 1341 | - | // tombstonesLocked renders the bounded tombstone publication, newest first. | |||
| 1342 | - | // Live endpoints are never published as tombstones. | |||
| 1343 | - | func (r *Registry) tombstonesLocked() []ports.BrokerHostTombstone { | |||
| 1344 | - | if len(r.tombstones) == 0 { | |||
| 1345 | - | return nil | |||
| 1346 | - | } | |||
| 1347 | - | out := make([]ports.BrokerHostTombstone, 0, len(r.tombstones)) | |||
| 1348 | - | for endpoint, tombstone := range r.tombstones { | |||
| 1349 | - | if _, live := r.hosts[endpoint]; live { | |||
| 1350 | - | continue | |||
| 1351 | - | } | |||
| 1352 | - | out = append(out, tombstone) | |||
| 1353 | - | } | |||
| 1354 | - | sort.Slice(out, func(i, j int) bool { | |||
| 1355 | - | if out[i].RetiredRevision != out[j].RetiredRevision { | |||
| 1356 | - | return out[i].RetiredRevision > out[j].RetiredRevision | |||
| 1357 | - | } | |||
| 1358 | - | return out[i].Endpoint < out[j].Endpoint | |||
| 1359 | - | }) | |||
| 1360 | - | if len(out) > ports.BrokerMaxTombstones { | |||
| 1361 | - | out = out[:ports.BrokerMaxTombstones] | |||
| 1362 | - | } | |||
| 1363 | - | return out | |||
| 1364 | - | } | |||
| 1365 | - | ||||
| 1366 | - | func (r *Registry) publishLocked(persist bool) { | |||
| 1367 | - | // Fail closed on an exhausted series: a wrapped revision would be zero, and | |||
| 1368 | - | // a wrapped series would publish revisions below the newest one, either of | |||
| 1369 | - | // which silently freezes durable persistence. The last valid snapshot stays | |||
| 1370 | - | // published, nothing is stored, and no subscriber is woken for a publication | |||
| 1371 | - | // that did not happen. | |||
| 1372 | - | if r.revisionExhaustedLocked() { | |||
| 1373 | - | return | |||
| 1374 | - | } | |||
| 1375 | - | // Remote daemons are published in registration order. The broker's own | |||
| 1376 | - | // machine daemon, when a producer is configured, is the | |||
| 1377 | - | // prepended entry at index zero; without a producer the registry produces no | |||
| 1378 | - | // local observation and never invents one. | |||
| 1379 | - | daemons := make([]ports.BrokerDaemonObservation, 0, len(r.hosts)+1) | |||
| 1380 | - | if r.local != nil { | |||
| 1381 | - | daemons = append(daemons, r.localHost.Clone()) | |||
| 1382 | - | } | |||
| 1383 | - | for _, endpoint := range r.order { | |||
| 1384 | - | host, ok := r.hosts[endpoint] | |||
| 1385 | - | if !ok { | |||
| 1386 | - | continue | |||
| 1387 | - | } | |||
| 1388 | - | daemons = append(daemons, host.Clone()) | |||
| 1389 | - | } | |||
| 1390 | - | snapshot := ports.BrokerSnapshot{Epoch: r.epoch, Revision: r.revision + 1, Daemons: daemons, Removed: r.tombstonesLocked()} | |||
| 1391 | - | if err := snapshot.Validate(); err != nil { | |||
| 1392 | - | // Fail closed: a publication the wire refuses aborts every client | |||
| 1393 | - | // connection on subscribe, so an invalid projection is never published. | |||
| 1394 | - | // The last valid snapshot stays current, nothing is persisted, and the | |||
| 1395 | - | // revision series does not advance past a publication that never | |||
| 1396 | - | // happened. | |||
| 1397 | - | r.log.Error("broker: refused invalid snapshot publication", "revision", r.revision+1, "err", err) | |||
| 1398 | - | return | |||
| 1399 | - | } | |||
| 1400 | - | r.revision = snapshot.Revision | |||
| 1401 | - | r.current.Store(&snapshot) | |||
| 1402 | - | if persist { | |||
| 1403 | - | // enqueue only stages an immutable publication: rendering the durable | |||
| 1404 | - | // copy and every store call happen on the writer goroutine, never | |||
| 1405 | - | // under the registry lock and never on the probe schedule. | |||
| 1406 | - | r.store.enqueue(snapshot) | |||
| 1407 | - | } | |||
| 1408 | - | for sub := range r.subs { | |||
| 1409 | - | sub.offer() | |||
| 1410 | - | } | |||
| 1411 | - | } | |||
| 1412 | - | ||||
| 1413 | - | // durable returns the persistable copy of a publication, rendered by the | |||
| 1414 | - | // writer goroutine with the shared rule (ports.BrokerSnapshot.DurableProjection). | |||
| 1415 | - | // The registry may publish the process-local daemon, which is never durable, | |||
| 1416 | - | // so it is dropped first; the store would refuse it otherwise. Restored | |||
| 1417 | - | // observations regain policy from membership (see stampHostAuthority). | |||
| 1418 | - | func durable(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |||
| 1419 | - | return snapshot.WithoutLocal().DurableProjection() | |||
| 1420 | - | } | |||
| 1421 | - | ||||
| 1422 | - | // snapshotWriter serializes durable snapshot writes off the registry lock. | |||
| 1423 | - | // Enqueue never blocks on store I/O and always coalesces to the newest | |||
| 1424 | - | // revision; Store calls run on one goroutine, so the durable store never sees | |||
| 1425 | - | // an out-of-order write. A failed write is logged and retried by the next | |||
| 1426 | - | // publication, so a store fault never stalls publication. Close flushes the | |||
| 1427 | - | // newest pending snapshot, stops the writer, and waits, so no write outlives | |||
| 1428 | - | // shutdown. | |||
| 1429 | - | type snapshotWriter struct { | |||
| 1430 | - | store ports.BrokerSnapshotStore | |||
| 1431 | - | log *slog.Logger | |||
| 1432 | - | ||||
| 1433 | - | mu sync.Mutex | |||
| 1434 | - | pending *ports.BrokerSnapshot | |||
| 1435 | - | written ports.BrokerRevision | |||
| 1436 | - | // last is the durable content of the newest successful write with its | |||
| 1437 | - | // revision and observation timestamps cleared. Only the writer goroutine | |||
| 1438 | - | // touches it. | |||
| 1439 | - | last *ports.BrokerSnapshot | |||
| 1440 | - | started bool | |||
| 1441 | - | closed bool | |||
| 1442 | - | wake chan struct{} | |||
| 1443 | - | stop chan struct{} | |||
| 1444 | - | done chan struct{} | |||
| 1445 | - | } | |||
| 1446 | - | ||||
| 1447 | - | func newSnapshotWriter(store ports.BrokerSnapshotStore, log *slog.Logger) *snapshotWriter { | |||
| 1448 | - | return &snapshotWriter{ | |||
| 1449 | - | store: store, log: log, | |||
| 1450 | - | wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), | |||
| 1451 | - | } | |||
| 1452 | - | } | |||
| 1453 | - | ||||
| 1454 | - | // enqueue stages a snapshot for durable persistence. It coalesces to the | |||
| 1455 | - | // newest revision, starts the writer on first use, and never blocks on the | |||
| 1456 | - | // store or copies session payloads. Enqueue after Close is refused: nothing is | |||
| 1457 | - | // written after shutdown. | |||
| 1458 | - | func (w *snapshotWriter) enqueue(snapshot ports.BrokerSnapshot) { | |||
| 1459 | - | w.mu.Lock() | |||
| 1460 | - | if w.closed { | |||
| 1461 | - | w.mu.Unlock() | |||
| 1462 | - | w.log.Debug("broker: durable snapshot write refused after shutdown", "revision", snapshot.Revision) | |||
| 1463 | - | return | |||
| 1464 | - | } | |||
| 1465 | - | if w.pending != nil { | |||
| 1466 | - | if snapshot.Revision <= w.pending.Revision { | |||
| 1467 | - | w.mu.Unlock() | |||
| 1468 | - | return | |||
| 1469 | - | } | |||
| 1470 | - | } else if snapshot.Revision <= w.written { | |||
| 1471 | - | w.mu.Unlock() | |||
| 1472 | - | return | |||
| 1473 | - | } | |||
| 1474 | - | // The publication is immutable after publication, so staging it is a value | |||
| 1475 | - | // copy; the defensive durable copy is rendered on the writer goroutine. | |||
| 1476 | - | w.pending = &snapshot | |||
| 1477 | - | if !w.started { | |||
| 1478 | - | w.started = true | |||
| 1479 | - | go w.run() | |||
| 1480 | - | } | |||
| 1481 | - | w.mu.Unlock() | |||
| 1482 | - | select { | |||
| 1483 | - | case w.wake <- struct{}{}: | |||
| 1484 | - | default: | |||
| 1485 | - | } | |||
| 1486 | - | } | |||
| 1487 | - | ||||
| 1488 | - | func (w *snapshotWriter) run() { | |||
| 1489 | - | defer close(w.done) | |||
| 1490 | - | for { | |||
| 1491 | - | select { | |||
| 1492 | - | case <-w.stop: | |||
| 1493 | - | w.flush() | |||
| 1494 | - | return | |||
| 1495 | - | case <-w.wake: | |||
| 1496 | - | w.flush() | |||
| 1497 | - | } | |||
| 1498 | - | } | |||
| 1499 | - | } | |||
| 1500 | - | ||||
| 1501 | - | // flush writes the newest staged snapshot, skipping revisions the store | |||
| 1502 | - | // already holds. | |||
| 1503 | - | func (w *snapshotWriter) flush() { | |||
| 1504 | - | for { | |||
| 1505 | - | w.mu.Lock() | |||
| 1506 | - | snapshot := w.pending | |||
| 1507 | - | written := w.written | |||
| 1508 | - | w.pending = nil | |||
| 1509 | - | w.mu.Unlock() | |||
| 1510 | - | if snapshot == nil { | |||
| 1511 | - | return | |||
| 1512 | - | } | |||
| 1513 | - | if snapshot.Revision <= written { | |||
| 1514 | - | continue | |||
| 1515 | - | } | |||
| 1516 | - | next := durable(*snapshot) | |||
| 1517 | - | key := durableContent(next) | |||
| 1518 | - | if w.last != nil && reflect.DeepEqual(*w.last, key) { | |||
| 1519 | - | // Only observation timestamps moved: the stored copy already | |||
| 1520 | - | // carries the same durable state, so skip the rewrite and fsync. | |||
| 1521 | - | // A restart re-observes every host before trusting freshness. | |||
| 1522 | - | continue | |||
| 1523 | - | } | |||
| 1524 | - | if err := w.store.Store(next); err != nil { | |||
| 1525 | - | w.last = nil | |||
| 1526 | - | w.log.Error("broker: durable snapshot write failed", "revision", snapshot.Revision, "err", err) | |||
| 1527 | - | continue | |||
| 1528 | - | } | |||
| 1529 | - | w.last = &key | |||
| 1530 | - | w.mu.Lock() | |||
| 1531 | - | if snapshot.Revision > w.written { | |||
| 1532 | - | w.written = snapshot.Revision | |||
| 1533 | - | } | |||
| 1534 | - | w.mu.Unlock() | |||
| 1535 | - | } | |||
| 1536 | - | } | |||
| 1537 | - | ||||
| 1538 | - | // durableContent returns the durable projection without the fields that change | |||
| 1539 | - | // on every routine observation: the revision and the attempt, success, and | |||
| 1540 | - | // due timestamps. | |||
| 1541 | - | func durableContent(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |||
| 1542 | - | out := snapshot | |||
| 1543 | - | out.Revision = 0 | |||
| 1544 | - | out.Daemons = make([]ports.BrokerDaemonObservation, len(snapshot.Daemons)) | |||
| 1545 | - | for i, daemon := range snapshot.Daemons { | |||
| 1546 | - | daemon.LastAttempt = time.Time{} | |||
| 1547 | - | daemon.LastSuccess = time.Time{} | |||
| 1548 | - | daemon.NextDue = time.Time{} | |||
| 1549 | - | out.Daemons[i] = daemon | |||
| 1550 | - | } | |||
| 1551 | - | return out | |||
| 1552 | - | } | |||
| 1553 | - | ||||
| 1554 | - | // close flushes the newest staged snapshot, stops the writer, and waits for | |||
| 1555 | - | // the drain to finish. It never returns before the in-flight write completes. | |||
| 1556 | - | func (w *snapshotWriter) close() { | |||
| 1557 | - | w.mu.Lock() | |||
| 1558 | - | if w.closed { | |||
| 1559 | - | w.mu.Unlock() | |||
| 1560 | - | return | |||
| 1561 | - | } | |||
| 1562 | - | w.closed = true | |||
| 1563 | - | started := w.started | |||
| 1564 | - | w.mu.Unlock() | |||
| 1565 | - | if !started { | |||
| 1566 | - | return | |||
| 1567 | - | } | |||
| 1568 | - | close(w.stop) | |||
| 1569 | - | <-w.done | |||
| 1570 | - | } | |||
| 1571 | - | ||||
| 1572 | - | type subscription struct { | |||
| 1573 | - | changed chan struct{} | |||
| 1574 | - | once sync.Once | |||
| 1575 | - | close func() | |||
| 1576 | - | } | |||
| 1577 | - | ||||
| 1578 | - | func (s *subscription) Changed() <-chan struct{} { return s.changed } | |||
| 1579 | - | func (s *subscription) Close() { s.once.Do(s.close) } | |||
| 1580 | - | func (s *subscription) offer() { | |||
| 1581 | - | select { | |||
| 1582 | - | case s.changed <- struct{}{}: | |||
| 1583 | - | default: | |||
| 1584 | - | } | |||
| 1585 | - | } | |||
A internal/usecase/broker/registry_membership.go +400 -0
| @@ -0,0 +1,400 @@ | |||
| 1 | + | package broker | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "errors" | |
| 6 | + | "fmt" | |
| 7 | + | "slices" | |
| 8 | + | "sort" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/domain" | |
| 11 | + | "github.com/bnema/vev/internal/ports" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | // AddHost adds or pins endpoint under policy. Duplicate same-policy additions | |
| 15 | + | // CAS-verify authority and preserve identity; a conflicting policy is refused. | |
| 16 | + | func (r *Registry) LookupHost(ctx context.Context, endpoint string) (ports.BrokerHostRecord, bool, error) { | |
| 17 | + | if err := ctx.Err(); err != nil { | |
| 18 | + | return ports.BrokerHostRecord{}, false, err | |
| 19 | + | } | |
| 20 | + | r.mu.Lock() | |
| 21 | + | defer r.mu.Unlock() | |
| 22 | + | for _, record := range r.authority.Hosts { | |
| 23 | + | if record.Registration.Endpoint == endpoint { | |
| 24 | + | record.Route = record.Route.Clone() | |
| 25 | + | return record, true, nil | |
| 26 | + | } | |
| 27 | + | } | |
| 28 | + | return ports.BrokerHostRecord{}, false, nil | |
| 29 | + | } | |
| 30 | + | ||
| 31 | + | func (r *Registry) AddHost(ctx context.Context, endpoint string, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |
| 32 | + | if err := domain.ValidateRemoteHostTarget(endpoint); err != nil { | |
| 33 | + | return domain.RemoteRegistration{}, err | |
| 34 | + | } | |
| 35 | + | if err := policy.Validate(); err != nil { | |
| 36 | + | return domain.RemoteRegistration{}, err | |
| 37 | + | } | |
| 38 | + | r.mu.Lock() | |
| 39 | + | defer r.mu.Unlock() | |
| 40 | + | if err := r.mutableLocked(ctx); err != nil { | |
| 41 | + | return domain.RemoteRegistration{}, err | |
| 42 | + | } | |
| 43 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 44 | + | for i := range records { | |
| 45 | + | if records[i].Registration.Endpoint != endpoint { | |
| 46 | + | continue | |
| 47 | + | } | |
| 48 | + | if records[i].Policy != policy { | |
| 49 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 50 | + | } | |
| 51 | + | records[i].Pinned = true | |
| 52 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 53 | + | return domain.RemoteRegistration{}, err | |
| 54 | + | } | |
| 55 | + | return records[i].Registration, nil | |
| 56 | + | } | |
| 57 | + | id, err := r.newIncarnation() | |
| 58 | + | if err != nil { | |
| 59 | + | return domain.RemoteRegistration{}, err | |
| 60 | + | } | |
| 61 | + | reg, err := domain.NewRemoteRegistration(endpoint, id) | |
| 62 | + | if err != nil { | |
| 63 | + | return domain.RemoteRegistration{}, err | |
| 64 | + | } | |
| 65 | + | if reg.IsZero() { | |
| 66 | + | return domain.RemoteRegistration{}, errors.New("broker: incarnation generator returned zero") | |
| 67 | + | } | |
| 68 | + | route, err := ports.BrokerRouteForTransport(policy.Transport, endpoint) | |
| 69 | + | if err != nil { | |
| 70 | + | return domain.RemoteRegistration{}, err | |
| 71 | + | } | |
| 72 | + | records = append(records, ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy, Route: route}) | |
| 73 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 74 | + | return domain.RemoteRegistration{}, err | |
| 75 | + | } | |
| 76 | + | return reg, nil | |
| 77 | + | } | |
| 78 | + | ||
| 79 | + | // RemoveHost removes only the exact expected registration. | |
| 80 | + | // BindAuthenticatedIdentity durably binds first-contact identity under the | |
| 81 | + | // exact registration and policy fence. The same binding is idempotent; a | |
| 82 | + | // different identity is refused and never overwrites authority. | |
| 83 | + | func (r *Registry) BindAuthenticatedIdentity(ctx context.Context, request ports.BrokerIdentityBindingRequest) (ports.BrokerDaemonIdentity, error) { | |
| 84 | + | if err := request.Validate(); err != nil { | |
| 85 | + | return "", err | |
| 86 | + | } | |
| 87 | + | if request.Fence.Local { | |
| 88 | + | return request.Identity, nil | |
| 89 | + | } | |
| 90 | + | if err := ctx.Err(); err != nil { | |
| 91 | + | return "", err | |
| 92 | + | } | |
| 93 | + | r.mu.Lock() | |
| 94 | + | defer r.mu.Unlock() | |
| 95 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 96 | + | for index := range records { | |
| 97 | + | record := &records[index] | |
| 98 | + | if record.Identity == request.Identity && record.Policy != request.Policy { | |
| 99 | + | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity is already bound under an incompatible policy")} | |
| 100 | + | } | |
| 101 | + | } | |
| 102 | + | for index := range records { | |
| 103 | + | record := &records[index] | |
| 104 | + | if !record.Registration.Equal(request.Fence.Registration) || record.Policy != request.Policy { | |
| 105 | + | continue | |
| 106 | + | } | |
| 107 | + | if record.Identity != "" { | |
| 108 | + | if record.Identity != request.Identity { | |
| 109 | + | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity conflicts with durable binding")} | |
| 110 | + | } | |
| 111 | + | return record.Identity, nil | |
| 112 | + | } | |
| 113 | + | record.Identity = request.Identity | |
| 114 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 115 | + | return "", err | |
| 116 | + | } | |
| 117 | + | return request.Identity, nil | |
| 118 | + | } | |
| 119 | + | return "", ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: errors.New("broker: stale identity binding fence")} | |
| 120 | + | } | |
| 121 | + | ||
| 122 | + | func (r *Registry) RemoveHost(ctx context.Context, expected domain.RemoteRegistration) (bool, error) { | |
| 123 | + | if err := expected.Validate(); err != nil { | |
| 124 | + | return false, err | |
| 125 | + | } | |
| 126 | + | r.mu.Lock() | |
| 127 | + | defer r.mu.Unlock() | |
| 128 | + | if err := r.mutableLocked(ctx); err != nil { | |
| 129 | + | return false, err | |
| 130 | + | } | |
| 131 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 132 | + | for i := range records { | |
| 133 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |
| 134 | + | continue | |
| 135 | + | } | |
| 136 | + | if !records[i].Registration.Equal(expected) { | |
| 137 | + | return false, ports.ErrBrokerHostConflict | |
| 138 | + | } | |
| 139 | + | records = append(records[:i], records[i+1:]...) | |
| 140 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 141 | + | return false, err | |
| 142 | + | } | |
| 143 | + | return true, nil | |
| 144 | + | } | |
| 145 | + | // A no-op still verifies the loaded revision. | |
| 146 | + | return false, r.commitMembershipLocked(records) | |
| 147 | + | } | |
| 148 | + | ||
| 149 | + | // UpdateHostPolicy updates exact authority and advances its generation. | |
| 150 | + | func (r *Registry) UpdateHostPolicy(ctx context.Context, expected domain.RemoteRegistration, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |
| 151 | + | if err := expected.Validate(); err != nil { | |
| 152 | + | return domain.RemoteRegistration{}, err | |
| 153 | + | } | |
| 154 | + | if err := policy.Validate(); err != nil { | |
| 155 | + | return domain.RemoteRegistration{}, err | |
| 156 | + | } | |
| 157 | + | r.mu.Lock() | |
| 158 | + | defer r.mu.Unlock() | |
| 159 | + | if err := r.mutableLocked(ctx); err != nil { | |
| 160 | + | return domain.RemoteRegistration{}, err | |
| 161 | + | } | |
| 162 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |
| 163 | + | for i := range records { | |
| 164 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |
| 165 | + | continue | |
| 166 | + | } | |
| 167 | + | if !records[i].Registration.Equal(expected) { | |
| 168 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 169 | + | } | |
| 170 | + | if records[i].Policy == policy { | |
| 171 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 172 | + | return domain.RemoteRegistration{}, err | |
| 173 | + | } | |
| 174 | + | return expected, nil | |
| 175 | + | } | |
| 176 | + | if expected.Generation == ^domain.RemoteGeneration(0) { | |
| 177 | + | return domain.RemoteRegistration{}, ports.ErrBrokerRevisionExhausted | |
| 178 | + | } | |
| 179 | + | route, err := ports.BrokerRouteForTransport(policy.Transport, expected.Endpoint) | |
| 180 | + | if err != nil { | |
| 181 | + | return domain.RemoteRegistration{}, err | |
| 182 | + | } | |
| 183 | + | records[i].Registration.Generation++ | |
| 184 | + | records[i].Route = route | |
| 185 | + | records[i].Policy = policy | |
| 186 | + | if err := r.commitMembershipLocked(records); err != nil { | |
| 187 | + | return domain.RemoteRegistration{}, err | |
| 188 | + | } | |
| 189 | + | return records[i].Registration, nil | |
| 190 | + | } | |
| 191 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |
| 192 | + | } | |
| 193 | + | ||
| 194 | + | func (r *Registry) mutableLocked(ctx context.Context) error { | |
| 195 | + | if r.membershipMode != MembershipMutable { | |
| 196 | + | return ports.ErrBrokerMembershipImmutable | |
| 197 | + | } | |
| 198 | + | if err := ctx.Err(); err != nil { | |
| 199 | + | return err | |
| 200 | + | } | |
| 201 | + | if r.running.Load() == runStopped { | |
| 202 | + | return ports.ErrBrokerRegistryClosed | |
| 203 | + | } | |
| 204 | + | if r.revisionExhaustedLocked() { | |
| 205 | + | return ports.ErrBrokerRevisionExhausted | |
| 206 | + | } | |
| 207 | + | return nil | |
| 208 | + | } | |
| 209 | + | ||
| 210 | + | // commitMembershipLocked is the sole mutable-membership durable commit path. | |
| 211 | + | // It installs projection only after CAS success. | |
| 212 | + | func (r *Registry) commitMembershipLocked(records []ports.BrokerHostRecord) error { | |
| 213 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 214 | + | return err | |
| 215 | + | } | |
| 216 | + | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | |
| 217 | + | return errors.New("broker: remote hosts require a host probe") | |
| 218 | + | } | |
| 219 | + | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | |
| 220 | + | order := make([]string, 0, len(records)) | |
| 221 | + | for rank, record := range records { | |
| 222 | + | observation := stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |
| 223 | + | // A membership the registry cannot publish is refused here, before the | |
| 224 | + | // store CAS, rather than persisted and left to freeze every later | |
| 225 | + | // publication: an endpoint whose derived display hint is empty after the | |
| 226 | + | // display rule drops its runes makes every snapshot carrying it invalid. | |
| 227 | + | if err := observation.Validate(); err != nil { | |
| 228 | + | return fmt.Errorf("broker: host record %q is not publishable: %w", record.Registration.Endpoint, err) | |
| 229 | + | } | |
| 230 | + | next[record.Registration.Endpoint] = observation | |
| 231 | + | order = append(order, record.Registration.Endpoint) | |
| 232 | + | } | |
| 233 | + | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | |
| 234 | + | return fmt.Errorf("broker: replace hosts: %w", err) | |
| 235 | + | } | |
| 236 | + | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: ports.CloneBrokerHostRecords(records)} | |
| 237 | + | r.setHostsLocked(next, order) | |
| 238 | + | return nil | |
| 239 | + | } | |
| 240 | + | ||
| 241 | + | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |
| 242 | + | // authority revision. It delegates to commitMembershipLocked, the same | |
| 243 | + | // validated commit path the runtime mutations use, so membership whose | |
| 244 | + | // projection could never be published is refused before the store CAS instead | |
| 245 | + | // of being persisted and silently freezing every later publication. Success | |
| 246 | + | // means membership is committed, not that advisory observations have flushed. | |
| 247 | + | // Store errors (including stale authority) are returned without changing the | |
| 248 | + | // projection, probe attempts, or local token. A settled run reports | |
| 249 | + | // ports.ErrBrokerRegistryClosed and an exhausted revision series reports | |
| 250 | + | // ports.ErrBrokerRevisionExhausted; both refuse the mutation before the store | |
| 251 | + | // CAS, so a refusal never changes durable authority either. Conflicts are not | |
| 252 | + | // retried or refreshed implicitly: reconcile by reopening the owner. Callers | |
| 253 | + | // must supply fresh incarnations when removing and re-adding. Even identical | |
| 254 | + | // records perform CAS so stale authority cannot report success. The synchronous | |
| 255 | + | // write holds mu; Snapshot remains lock-free, but scheduling waits for | |
| 256 | + | // membership durability. Observation writes remain asynchronous. | |
| 257 | + | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |
| 258 | + | var err error | |
| 259 | + | records, err = ports.UpgradeBrokerHostRoutes(records) | |
| 260 | + | if err != nil { | |
| 261 | + | return err | |
| 262 | + | } | |
| 263 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |
| 264 | + | return err | |
| 265 | + | } | |
| 266 | + | r.mu.Lock() | |
| 267 | + | defer r.mu.Unlock() | |
| 268 | + | if r.running.Load() == runStopped { | |
| 269 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |
| 270 | + | } | |
| 271 | + | if r.revisionExhaustedLocked() { | |
| 272 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |
| 273 | + | } | |
| 274 | + | return r.commitMembershipLocked(records) | |
| 275 | + | } | |
| 276 | + | ||
| 277 | + | // setHostsLocked applies only a projection; the public path commits authority | |
| 278 | + | // first. The caller holds mu through both steps, fencing probe completions. | |
| 279 | + | // order is the registration order the projection was built from; it becomes | |
| 280 | + | // the publication order of remote daemons. | |
| 281 | + | func (r *Registry) setHostsLocked(next map[string]ports.BrokerDaemonObservation, order []string) { | |
| 282 | + | if registrationsUnchanged(r.hosts, next, r.order, order) { | |
| 283 | + | return | |
| 284 | + | } | |
| 285 | + | for endpoint, current := range r.hosts { | |
| 286 | + | candidate, present := next[endpoint] | |
| 287 | + | if present && current.Registration.Equal(candidate.Registration) && current.Policy == candidate.Policy { | |
| 288 | + | // Unchanged authority: keep the live observation as is. Rank is the | |
| 289 | + | // one authority field that follows membership order rather than the | |
| 290 | + | // record, so it is re-stamped from the new projection. | |
| 291 | + | kept := current | |
| 292 | + | kept.Rank = candidate.Rank | |
| 293 | + | next[endpoint] = kept | |
| 294 | + | continue | |
| 295 | + | } | |
| 296 | + | // A retired registration owns no observation: cancelling its attempt | |
| 297 | + | // releases a ctx-honoring probe immediately, and dropping its record | |
| 298 | + | // means the retired completion can never clear or answer the | |
| 299 | + | // replacement, and the replacement is never blocked behind it. The | |
| 300 | + | // cancellation happens under the registry lock, so no replacement | |
| 301 | + | // attempt can start before the retired context is cancelled. | |
| 302 | + | r.cancelInflightLocked(endpoint) | |
| 303 | + | delete(r.pending, endpoint) | |
| 304 | + | if present { | |
| 305 | + | // The endpoint stays live under a new registration, so its | |
| 306 | + | // retirement is never published as a tombstone alongside a live | |
| 307 | + | // host; the attempt token fences the stale completion. | |
| 308 | + | delete(r.tombstones, endpoint) | |
| 309 | + | continue | |
| 310 | + | } | |
| 311 | + | r.retireLocked(current.Registration) | |
| 312 | + | } | |
| 313 | + | for endpoint := range next { | |
| 314 | + | // Re-adding an endpoint supersedes its tombstone: a live host and a | |
| 315 | + | // tombstone for the same endpoint are mutually exclusive. | |
| 316 | + | delete(r.tombstones, endpoint) | |
| 317 | + | } | |
| 318 | + | r.hosts = next | |
| 319 | + | r.order = order | |
| 320 | + | r.publishLocked(true) | |
| 321 | + | r.hint() | |
| 322 | + | } | |
| 323 | + | ||
| 324 | + | // registrationsUnchanged reports exact membership equality: the same endpoints | |
| 325 | + | // in the same registration order, bound to the same registration identities and | |
| 326 | + | // policy. Publication order and presentation rank are functions of durable | |
| 327 | + | // membership order, so a reordering of identical records is a membership change | |
| 328 | + | // that must republish; otherwise the same durable state yields different | |
| 329 | + | // publications depending on process history. Projection fields such as | |
| 330 | + | // availability, checking, or inventory never participate. | |
| 331 | + | func registrationsUnchanged(current, next map[string]ports.BrokerDaemonObservation, currentOrder, nextOrder []string) bool { | |
| 332 | + | if len(current) != len(next) { | |
| 333 | + | return false | |
| 334 | + | } | |
| 335 | + | if !slices.Equal(currentOrder, nextOrder) { | |
| 336 | + | return false | |
| 337 | + | } | |
| 338 | + | for endpoint, candidate := range next { | |
| 339 | + | existing, ok := current[endpoint] | |
| 340 | + | if !ok || !existing.Registration.Equal(candidate.Registration) || existing.Policy != candidate.Policy { | |
| 341 | + | return false | |
| 342 | + | } | |
| 343 | + | } | |
| 344 | + | return true | |
| 345 | + | } | |
| 346 | + | ||
| 347 | + | // retireLocked records the tombstone for one removed registration at the | |
| 348 | + | // revision of the publication that carries it. Mutation paths refuse an | |
| 349 | + | // exhausted series before retiring, so the retirement revision can never wrap | |
| 350 | + | // to zero or fall below the last published one. | |
| 351 | + | func (r *Registry) retireLocked(registration domain.RemoteRegistration) { | |
| 352 | + | retired := r.revision + 1 | |
| 353 | + | r.tombstones[registration.Endpoint] = ports.BrokerHostTombstone{ | |
| 354 | + | Endpoint: registration.Endpoint, | |
| 355 | + | Registration: registration, | |
| 356 | + | RetiredRevision: retired, | |
| 357 | + | } | |
| 358 | + | r.pruneTombstonesLocked() | |
| 359 | + | } | |
| 360 | + | ||
| 361 | + | // pruneTombstonesLocked keeps only the newest BrokerMaxTombstones retirements | |
| 362 | + | // so an endpoint that churns can never grow the published set without bound. | |
| 363 | + | func (r *Registry) pruneTombstonesLocked() { | |
| 364 | + | for len(r.tombstones) > ports.BrokerMaxTombstones { | |
| 365 | + | oldest := "" | |
| 366 | + | var oldestRevision ports.BrokerRevision | |
| 367 | + | for endpoint, tombstone := range r.tombstones { | |
| 368 | + | if oldest == "" || tombstone.RetiredRevision < oldestRevision || | |
| 369 | + | (tombstone.RetiredRevision == oldestRevision && endpoint < oldest) { | |
| 370 | + | oldest, oldestRevision = endpoint, tombstone.RetiredRevision | |
| 371 | + | } | |
| 372 | + | } | |
| 373 | + | delete(r.tombstones, oldest) | |
| 374 | + | } | |
| 375 | + | } | |
| 376 | + | ||
| 377 | + | // tombstonesLocked renders the bounded tombstone publication, newest first. | |
| 378 | + | // Live endpoints are never published as tombstones. | |
| 379 | + | func (r *Registry) tombstonesLocked() []ports.BrokerHostTombstone { | |
| 380 | + | if len(r.tombstones) == 0 { | |
| 381 | + | return nil | |
| 382 | + | } | |
| 383 | + | out := make([]ports.BrokerHostTombstone, 0, len(r.tombstones)) | |
| 384 | + | for endpoint, tombstone := range r.tombstones { | |
| 385 | + | if _, live := r.hosts[endpoint]; live { | |
| 386 | + | continue | |
| 387 | + | } | |
| 388 | + | out = append(out, tombstone) | |
| 389 | + | } | |
| 390 | + | sort.Slice(out, func(i, j int) bool { | |
| 391 | + | if out[i].RetiredRevision != out[j].RetiredRevision { | |
| 392 | + | return out[i].RetiredRevision > out[j].RetiredRevision | |
| 393 | + | } | |
| 394 | + | return out[i].Endpoint < out[j].Endpoint | |
| 395 | + | }) | |
| 396 | + | if len(out) > ports.BrokerMaxTombstones { | |
| 397 | + | out = out[:ports.BrokerMaxTombstones] | |
| 398 | + | } | |
| 399 | + | return out | |
| 400 | + | } | |
| @@ -0,0 +1,400 @@ | |||||
| 1 | + | package broker | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "errors" | |||
| 6 | + | "fmt" | |||
| 7 | + | "slices" | |||
| 8 | + | "sort" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | // AddHost adds or pins endpoint under policy. Duplicate same-policy additions | |||
| 15 | + | // CAS-verify authority and preserve identity; a conflicting policy is refused. | |||
| 16 | + | func (r *Registry) LookupHost(ctx context.Context, endpoint string) (ports.BrokerHostRecord, bool, error) { | |||
| 17 | + | if err := ctx.Err(); err != nil { | |||
| 18 | + | return ports.BrokerHostRecord{}, false, err | |||
| 19 | + | } | |||
| 20 | + | r.mu.Lock() | |||
| 21 | + | defer r.mu.Unlock() | |||
| 22 | + | for _, record := range r.authority.Hosts { | |||
| 23 | + | if record.Registration.Endpoint == endpoint { | |||
| 24 | + | record.Route = record.Route.Clone() | |||
| 25 | + | return record, true, nil | |||
| 26 | + | } | |||
| 27 | + | } | |||
| 28 | + | return ports.BrokerHostRecord{}, false, nil | |||
| 29 | + | } | |||
| 30 | + | ||||
| 31 | + | func (r *Registry) AddHost(ctx context.Context, endpoint string, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |||
| 32 | + | if err := domain.ValidateRemoteHostTarget(endpoint); err != nil { | |||
| 33 | + | return domain.RemoteRegistration{}, err | |||
| 34 | + | } | |||
| 35 | + | if err := policy.Validate(); err != nil { | |||
| 36 | + | return domain.RemoteRegistration{}, err | |||
| 37 | + | } | |||
| 38 | + | r.mu.Lock() | |||
| 39 | + | defer r.mu.Unlock() | |||
| 40 | + | if err := r.mutableLocked(ctx); err != nil { | |||
| 41 | + | return domain.RemoteRegistration{}, err | |||
| 42 | + | } | |||
| 43 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 44 | + | for i := range records { | |||
| 45 | + | if records[i].Registration.Endpoint != endpoint { | |||
| 46 | + | continue | |||
| 47 | + | } | |||
| 48 | + | if records[i].Policy != policy { | |||
| 49 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 50 | + | } | |||
| 51 | + | records[i].Pinned = true | |||
| 52 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 53 | + | return domain.RemoteRegistration{}, err | |||
| 54 | + | } | |||
| 55 | + | return records[i].Registration, nil | |||
| 56 | + | } | |||
| 57 | + | id, err := r.newIncarnation() | |||
| 58 | + | if err != nil { | |||
| 59 | + | return domain.RemoteRegistration{}, err | |||
| 60 | + | } | |||
| 61 | + | reg, err := domain.NewRemoteRegistration(endpoint, id) | |||
| 62 | + | if err != nil { | |||
| 63 | + | return domain.RemoteRegistration{}, err | |||
| 64 | + | } | |||
| 65 | + | if reg.IsZero() { | |||
| 66 | + | return domain.RemoteRegistration{}, errors.New("broker: incarnation generator returned zero") | |||
| 67 | + | } | |||
| 68 | + | route, err := ports.BrokerRouteForTransport(policy.Transport, endpoint) | |||
| 69 | + | if err != nil { | |||
| 70 | + | return domain.RemoteRegistration{}, err | |||
| 71 | + | } | |||
| 72 | + | records = append(records, ports.BrokerHostRecord{Registration: reg, Pinned: true, Policy: policy, Route: route}) | |||
| 73 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 74 | + | return domain.RemoteRegistration{}, err | |||
| 75 | + | } | |||
| 76 | + | return reg, nil | |||
| 77 | + | } | |||
| 78 | + | ||||
| 79 | + | // RemoveHost removes only the exact expected registration. | |||
| 80 | + | // BindAuthenticatedIdentity durably binds first-contact identity under the | |||
| 81 | + | // exact registration and policy fence. The same binding is idempotent; a | |||
| 82 | + | // different identity is refused and never overwrites authority. | |||
| 83 | + | func (r *Registry) BindAuthenticatedIdentity(ctx context.Context, request ports.BrokerIdentityBindingRequest) (ports.BrokerDaemonIdentity, error) { | |||
| 84 | + | if err := request.Validate(); err != nil { | |||
| 85 | + | return "", err | |||
| 86 | + | } | |||
| 87 | + | if request.Fence.Local { | |||
| 88 | + | return request.Identity, nil | |||
| 89 | + | } | |||
| 90 | + | if err := ctx.Err(); err != nil { | |||
| 91 | + | return "", err | |||
| 92 | + | } | |||
| 93 | + | r.mu.Lock() | |||
| 94 | + | defer r.mu.Unlock() | |||
| 95 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 96 | + | for index := range records { | |||
| 97 | + | record := &records[index] | |||
| 98 | + | if record.Identity == request.Identity && record.Policy != request.Policy { | |||
| 99 | + | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity is already bound under an incompatible policy")} | |||
| 100 | + | } | |||
| 101 | + | } | |||
| 102 | + | for index := range records { | |||
| 103 | + | record := &records[index] | |||
| 104 | + | if !record.Registration.Equal(request.Fence.Registration) || record.Policy != request.Policy { | |||
| 105 | + | continue | |||
| 106 | + | } | |||
| 107 | + | if record.Identity != "" { | |||
| 108 | + | if record.Identity != request.Identity { | |||
| 109 | + | return "", ports.BrokerError{Code: ports.BrokerErrorConflictingPolicy, Cause: errors.New("broker: authenticated daemon identity conflicts with durable binding")} | |||
| 110 | + | } | |||
| 111 | + | return record.Identity, nil | |||
| 112 | + | } | |||
| 113 | + | record.Identity = request.Identity | |||
| 114 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 115 | + | return "", err | |||
| 116 | + | } | |||
| 117 | + | return request.Identity, nil | |||
| 118 | + | } | |||
| 119 | + | return "", ports.BrokerError{Code: ports.BrokerErrorUnavailable, Cause: errors.New("broker: stale identity binding fence")} | |||
| 120 | + | } | |||
| 121 | + | ||||
| 122 | + | func (r *Registry) RemoveHost(ctx context.Context, expected domain.RemoteRegistration) (bool, error) { | |||
| 123 | + | if err := expected.Validate(); err != nil { | |||
| 124 | + | return false, err | |||
| 125 | + | } | |||
| 126 | + | r.mu.Lock() | |||
| 127 | + | defer r.mu.Unlock() | |||
| 128 | + | if err := r.mutableLocked(ctx); err != nil { | |||
| 129 | + | return false, err | |||
| 130 | + | } | |||
| 131 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 132 | + | for i := range records { | |||
| 133 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |||
| 134 | + | continue | |||
| 135 | + | } | |||
| 136 | + | if !records[i].Registration.Equal(expected) { | |||
| 137 | + | return false, ports.ErrBrokerHostConflict | |||
| 138 | + | } | |||
| 139 | + | records = append(records[:i], records[i+1:]...) | |||
| 140 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 141 | + | return false, err | |||
| 142 | + | } | |||
| 143 | + | return true, nil | |||
| 144 | + | } | |||
| 145 | + | // A no-op still verifies the loaded revision. | |||
| 146 | + | return false, r.commitMembershipLocked(records) | |||
| 147 | + | } | |||
| 148 | + | ||||
| 149 | + | // UpdateHostPolicy updates exact authority and advances its generation. | |||
| 150 | + | func (r *Registry) UpdateHostPolicy(ctx context.Context, expected domain.RemoteRegistration, policy ports.BrokerPolicy) (domain.RemoteRegistration, error) { | |||
| 151 | + | if err := expected.Validate(); err != nil { | |||
| 152 | + | return domain.RemoteRegistration{}, err | |||
| 153 | + | } | |||
| 154 | + | if err := policy.Validate(); err != nil { | |||
| 155 | + | return domain.RemoteRegistration{}, err | |||
| 156 | + | } | |||
| 157 | + | r.mu.Lock() | |||
| 158 | + | defer r.mu.Unlock() | |||
| 159 | + | if err := r.mutableLocked(ctx); err != nil { | |||
| 160 | + | return domain.RemoteRegistration{}, err | |||
| 161 | + | } | |||
| 162 | + | records := append([]ports.BrokerHostRecord(nil), r.authority.Hosts...) | |||
| 163 | + | for i := range records { | |||
| 164 | + | if records[i].Registration.Endpoint != expected.Endpoint { | |||
| 165 | + | continue | |||
| 166 | + | } | |||
| 167 | + | if !records[i].Registration.Equal(expected) { | |||
| 168 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 169 | + | } | |||
| 170 | + | if records[i].Policy == policy { | |||
| 171 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 172 | + | return domain.RemoteRegistration{}, err | |||
| 173 | + | } | |||
| 174 | + | return expected, nil | |||
| 175 | + | } | |||
| 176 | + | if expected.Generation == ^domain.RemoteGeneration(0) { | |||
| 177 | + | return domain.RemoteRegistration{}, ports.ErrBrokerRevisionExhausted | |||
| 178 | + | } | |||
| 179 | + | route, err := ports.BrokerRouteForTransport(policy.Transport, expected.Endpoint) | |||
| 180 | + | if err != nil { | |||
| 181 | + | return domain.RemoteRegistration{}, err | |||
| 182 | + | } | |||
| 183 | + | records[i].Registration.Generation++ | |||
| 184 | + | records[i].Route = route | |||
| 185 | + | records[i].Policy = policy | |||
| 186 | + | if err := r.commitMembershipLocked(records); err != nil { | |||
| 187 | + | return domain.RemoteRegistration{}, err | |||
| 188 | + | } | |||
| 189 | + | return records[i].Registration, nil | |||
| 190 | + | } | |||
| 191 | + | return domain.RemoteRegistration{}, ports.ErrBrokerHostConflict | |||
| 192 | + | } | |||
| 193 | + | ||||
| 194 | + | func (r *Registry) mutableLocked(ctx context.Context) error { | |||
| 195 | + | if r.membershipMode != MembershipMutable { | |||
| 196 | + | return ports.ErrBrokerMembershipImmutable | |||
| 197 | + | } | |||
| 198 | + | if err := ctx.Err(); err != nil { | |||
| 199 | + | return err | |||
| 200 | + | } | |||
| 201 | + | if r.running.Load() == runStopped { | |||
| 202 | + | return ports.ErrBrokerRegistryClosed | |||
| 203 | + | } | |||
| 204 | + | if r.revisionExhaustedLocked() { | |||
| 205 | + | return ports.ErrBrokerRevisionExhausted | |||
| 206 | + | } | |||
| 207 | + | return nil | |||
| 208 | + | } | |||
| 209 | + | ||||
| 210 | + | // commitMembershipLocked is the sole mutable-membership durable commit path. | |||
| 211 | + | // It installs projection only after CAS success. | |||
| 212 | + | func (r *Registry) commitMembershipLocked(records []ports.BrokerHostRecord) error { | |||
| 213 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 214 | + | return err | |||
| 215 | + | } | |||
| 216 | + | if len(records) > 0 && !r.observationDisabled && nilDependency(r.probe) { | |||
| 217 | + | return errors.New("broker: remote hosts require a host probe") | |||
| 218 | + | } | |||
| 219 | + | next := make(map[string]ports.BrokerDaemonObservation, len(records)) | |||
| 220 | + | order := make([]string, 0, len(records)) | |||
| 221 | + | for rank, record := range records { | |||
| 222 | + | observation := stampHostAuthority(ports.BrokerDaemonObservation{}, record, rank) | |||
| 223 | + | // A membership the registry cannot publish is refused here, before the | |||
| 224 | + | // store CAS, rather than persisted and left to freeze every later | |||
| 225 | + | // publication: an endpoint whose derived display hint is empty after the | |||
| 226 | + | // display rule drops its runes makes every snapshot carrying it invalid. | |||
| 227 | + | if err := observation.Validate(); err != nil { | |||
| 228 | + | return fmt.Errorf("broker: host record %q is not publishable: %w", record.Registration.Endpoint, err) | |||
| 229 | + | } | |||
| 230 | + | next[record.Registration.Endpoint] = observation | |||
| 231 | + | order = append(order, record.Registration.Endpoint) | |||
| 232 | + | } | |||
| 233 | + | if err := r.hostStore.ReplaceHosts(r.authority.Revision, records); err != nil { | |||
| 234 | + | return fmt.Errorf("broker: replace hosts: %w", err) | |||
| 235 | + | } | |||
| 236 | + | r.authority = ports.BrokerHosts{Revision: r.authority.Revision + 1, Hosts: ports.CloneBrokerHostRecords(records)} | |||
| 237 | + | r.setHostsLocked(next, order) | |||
| 238 | + | return nil | |||
| 239 | + | } | |||
| 240 | + | ||||
| 241 | + | // ReplaceHosts durably replaces membership and explicit policy using the loaded | |||
| 242 | + | // authority revision. It delegates to commitMembershipLocked, the same | |||
| 243 | + | // validated commit path the runtime mutations use, so membership whose | |||
| 244 | + | // projection could never be published is refused before the store CAS instead | |||
| 245 | + | // of being persisted and silently freezing every later publication. Success | |||
| 246 | + | // means membership is committed, not that advisory observations have flushed. | |||
| 247 | + | // Store errors (including stale authority) are returned without changing the | |||
| 248 | + | // projection, probe attempts, or local token. A settled run reports | |||
| 249 | + | // ports.ErrBrokerRegistryClosed and an exhausted revision series reports | |||
| 250 | + | // ports.ErrBrokerRevisionExhausted; both refuse the mutation before the store | |||
| 251 | + | // CAS, so a refusal never changes durable authority either. Conflicts are not | |||
| 252 | + | // retried or refreshed implicitly: reconcile by reopening the owner. Callers | |||
| 253 | + | // must supply fresh incarnations when removing and re-adding. Even identical | |||
| 254 | + | // records perform CAS so stale authority cannot report success. The synchronous | |||
| 255 | + | // write holds mu; Snapshot remains lock-free, but scheduling waits for | |||
| 256 | + | // membership durability. Observation writes remain asynchronous. | |||
| 257 | + | func (r *Registry) ReplaceHosts(records []ports.BrokerHostRecord) error { | |||
| 258 | + | var err error | |||
| 259 | + | records, err = ports.UpgradeBrokerHostRoutes(records) | |||
| 260 | + | if err != nil { | |||
| 261 | + | return err | |||
| 262 | + | } | |||
| 263 | + | if err := ports.ValidateBrokerHostRecords(records); err != nil { | |||
| 264 | + | return err | |||
| 265 | + | } | |||
| 266 | + | r.mu.Lock() | |||
| 267 | + | defer r.mu.Unlock() | |||
| 268 | + | if r.running.Load() == runStopped { | |||
| 269 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRegistryClosed) | |||
| 270 | + | } | |||
| 271 | + | if r.revisionExhaustedLocked() { | |||
| 272 | + | return fmt.Errorf("broker: membership replacement: %w", ports.ErrBrokerRevisionExhausted) | |||
| 273 | + | } | |||
| 274 | + | return r.commitMembershipLocked(records) | |||
| 275 | + | } | |||
| 276 | + | ||||
| 277 | + | // setHostsLocked applies only a projection; the public path commits authority | |||
| 278 | + | // first. The caller holds mu through both steps, fencing probe completions. | |||
| 279 | + | // order is the registration order the projection was built from; it becomes | |||
| 280 | + | // the publication order of remote daemons. | |||
| 281 | + | func (r *Registry) setHostsLocked(next map[string]ports.BrokerDaemonObservation, order []string) { | |||
| 282 | + | if registrationsUnchanged(r.hosts, next, r.order, order) { | |||
| 283 | + | return | |||
| 284 | + | } | |||
| 285 | + | for endpoint, current := range r.hosts { | |||
| 286 | + | candidate, present := next[endpoint] | |||
| 287 | + | if present && current.Registration.Equal(candidate.Registration) && current.Policy == candidate.Policy { | |||
| 288 | + | // Unchanged authority: keep the live observation as is. Rank is the | |||
| 289 | + | // one authority field that follows membership order rather than the | |||
| 290 | + | // record, so it is re-stamped from the new projection. | |||
| 291 | + | kept := current | |||
| 292 | + | kept.Rank = candidate.Rank | |||
| 293 | + | next[endpoint] = kept | |||
| 294 | + | continue | |||
| 295 | + | } | |||
| 296 | + | // A retired registration owns no observation: cancelling its attempt | |||
| 297 | + | // releases a ctx-honoring probe immediately, and dropping its record | |||
| 298 | + | // means the retired completion can never clear or answer the | |||
| 299 | + | // replacement, and the replacement is never blocked behind it. The | |||
| 300 | + | // cancellation happens under the registry lock, so no replacement | |||
| 301 | + | // attempt can start before the retired context is cancelled. | |||
| 302 | + | r.cancelInflightLocked(endpoint) | |||
| 303 | + | delete(r.pending, endpoint) | |||
| 304 | + | if present { | |||
| 305 | + | // The endpoint stays live under a new registration, so its | |||
| 306 | + | // retirement is never published as a tombstone alongside a live | |||
| 307 | + | // host; the attempt token fences the stale completion. | |||
| 308 | + | delete(r.tombstones, endpoint) | |||
| 309 | + | continue | |||
| 310 | + | } | |||
| 311 | + | r.retireLocked(current.Registration) | |||
| 312 | + | } | |||
| 313 | + | for endpoint := range next { | |||
| 314 | + | // Re-adding an endpoint supersedes its tombstone: a live host and a | |||
| 315 | + | // tombstone for the same endpoint are mutually exclusive. | |||
| 316 | + | delete(r.tombstones, endpoint) | |||
| 317 | + | } | |||
| 318 | + | r.hosts = next | |||
| 319 | + | r.order = order | |||
| 320 | + | r.publishLocked(true) | |||
| 321 | + | r.hint() | |||
| 322 | + | } | |||
| 323 | + | ||||
| 324 | + | // registrationsUnchanged reports exact membership equality: the same endpoints | |||
| 325 | + | // in the same registration order, bound to the same registration identities and | |||
| 326 | + | // policy. Publication order and presentation rank are functions of durable | |||
| 327 | + | // membership order, so a reordering of identical records is a membership change | |||
| 328 | + | // that must republish; otherwise the same durable state yields different | |||
| 329 | + | // publications depending on process history. Projection fields such as | |||
| 330 | + | // availability, checking, or inventory never participate. | |||
| 331 | + | func registrationsUnchanged(current, next map[string]ports.BrokerDaemonObservation, currentOrder, nextOrder []string) bool { | |||
| 332 | + | if len(current) != len(next) { | |||
| 333 | + | return false | |||
| 334 | + | } | |||
| 335 | + | if !slices.Equal(currentOrder, nextOrder) { | |||
| 336 | + | return false | |||
| 337 | + | } | |||
| 338 | + | for endpoint, candidate := range next { | |||
| 339 | + | existing, ok := current[endpoint] | |||
| 340 | + | if !ok || !existing.Registration.Equal(candidate.Registration) || existing.Policy != candidate.Policy { | |||
| 341 | + | return false | |||
| 342 | + | } | |||
| 343 | + | } | |||
| 344 | + | return true | |||
| 345 | + | } | |||
| 346 | + | ||||
| 347 | + | // retireLocked records the tombstone for one removed registration at the | |||
| 348 | + | // revision of the publication that carries it. Mutation paths refuse an | |||
| 349 | + | // exhausted series before retiring, so the retirement revision can never wrap | |||
| 350 | + | // to zero or fall below the last published one. | |||
| 351 | + | func (r *Registry) retireLocked(registration domain.RemoteRegistration) { | |||
| 352 | + | retired := r.revision + 1 | |||
| 353 | + | r.tombstones[registration.Endpoint] = ports.BrokerHostTombstone{ | |||
| 354 | + | Endpoint: registration.Endpoint, | |||
| 355 | + | Registration: registration, | |||
| 356 | + | RetiredRevision: retired, | |||
| 357 | + | } | |||
| 358 | + | r.pruneTombstonesLocked() | |||
| 359 | + | } | |||
| 360 | + | ||||
| 361 | + | // pruneTombstonesLocked keeps only the newest BrokerMaxTombstones retirements | |||
| 362 | + | // so an endpoint that churns can never grow the published set without bound. | |||
| 363 | + | func (r *Registry) pruneTombstonesLocked() { | |||
| 364 | + | for len(r.tombstones) > ports.BrokerMaxTombstones { | |||
| 365 | + | oldest := "" | |||
| 366 | + | var oldestRevision ports.BrokerRevision | |||
| 367 | + | for endpoint, tombstone := range r.tombstones { | |||
| 368 | + | if oldest == "" || tombstone.RetiredRevision < oldestRevision || | |||
| 369 | + | (tombstone.RetiredRevision == oldestRevision && endpoint < oldest) { | |||
| 370 | + | oldest, oldestRevision = endpoint, tombstone.RetiredRevision | |||
| 371 | + | } | |||
| 372 | + | } | |||
| 373 | + | delete(r.tombstones, oldest) | |||
| 374 | + | } | |||
| 375 | + | } | |||
| 376 | + | ||||
| 377 | + | // tombstonesLocked renders the bounded tombstone publication, newest first. | |||
| 378 | + | // Live endpoints are never published as tombstones. | |||
| 379 | + | func (r *Registry) tombstonesLocked() []ports.BrokerHostTombstone { | |||
| 380 | + | if len(r.tombstones) == 0 { | |||
| 381 | + | return nil | |||
| 382 | + | } | |||
| 383 | + | out := make([]ports.BrokerHostTombstone, 0, len(r.tombstones)) | |||
| 384 | + | for endpoint, tombstone := range r.tombstones { | |||
| 385 | + | if _, live := r.hosts[endpoint]; live { | |||
| 386 | + | continue | |||
| 387 | + | } | |||
| 388 | + | out = append(out, tombstone) | |||
| 389 | + | } | |||
| 390 | + | sort.Slice(out, func(i, j int) bool { | |||
| 391 | + | if out[i].RetiredRevision != out[j].RetiredRevision { | |||
| 392 | + | return out[i].RetiredRevision > out[j].RetiredRevision | |||
| 393 | + | } | |||
| 394 | + | return out[i].Endpoint < out[j].Endpoint | |||
| 395 | + | }) | |||
| 396 | + | if len(out) > ports.BrokerMaxTombstones { | |||
| 397 | + | out = out[:ports.BrokerMaxTombstones] | |||
| 398 | + | } | |||
| 399 | + | return out | |||
| 400 | + | } | |||
A internal/usecase/broker/registry_probe.go +528 -0
| @@ -0,0 +1,528 @@ | |||
| 1 | + | package broker | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "context" | |
| 5 | + | "encoding/binary" | |
| 6 | + | "errors" | |
| 7 | + | "hash/fnv" | |
| 8 | + | "time" | |
| 9 | + | ||
| 10 | + | "github.com/bnema/vev/internal/domain" | |
| 11 | + | "github.com/bnema/vev/internal/ports" | |
| 12 | + | ) | |
| 13 | + | ||
| 14 | + | // cancelInflightLocked cancels the in-flight attempt for one endpoint and | |
| 15 | + | // drops its record. Callers must hold r.mu: the retired attempt's context is | |
| 16 | + | // cancelled before the record disappears, so no replacement attempt can start | |
| 17 | + | // while the retired probe's context is still live. | |
| 18 | + | func (r *Registry) cancelInflightLocked(endpoint string) { | |
| 19 | + | if attempt := r.inflight[endpoint]; attempt != nil { | |
| 20 | + | attempt.cancel() | |
| 21 | + | delete(r.inflight, endpoint) | |
| 22 | + | } | |
| 23 | + | } | |
| 24 | + | ||
| 25 | + | // cancelAllInflightLocked cancels every in-flight attempt and drops the | |
| 26 | + | // records. Callers must hold r.mu. | |
| 27 | + | func (r *Registry) cancelAllInflightLocked() { | |
| 28 | + | for endpoint, attempt := range r.inflight { | |
| 29 | + | attempt.cancel() | |
| 30 | + | delete(r.inflight, endpoint) | |
| 31 | + | } | |
| 32 | + | } | |
| 33 | + | ||
| 34 | + | // RequestProbe is non-blocking and coalesces concurrent demand per endpoint. | |
| 35 | + | // A disabled registry never observes, so the demand is dropped without | |
| 36 | + | // scheduling anything. An empty endpoint requests a local re-probe instead of a | |
| 37 | + | // configured remote: the broker's own machine daemon is re-observed ahead of | |
| 38 | + | // its scheduled freshness window, honored by dispatchLocalLocked exactly like | |
| 39 | + | // pending honors remote demand. A registry with no local observation producer | |
| 40 | + | // drops it, just like an unknown remote endpoint is dropped. | |
| 41 | + | func (r *Registry) RequestProbe(endpoint string) { | |
| 42 | + | if r.observationDisabled { | |
| 43 | + | return | |
| 44 | + | } | |
| 45 | + | r.mu.Lock() | |
| 46 | + | if endpoint == "" { | |
| 47 | + | if r.local != nil { | |
| 48 | + | r.localPending = true | |
| 49 | + | } | |
| 50 | + | } else if _, ok := r.hosts[endpoint]; ok { | |
| 51 | + | r.pending[endpoint] = true | |
| 52 | + | } | |
| 53 | + | r.mu.Unlock() | |
| 54 | + | r.hint() | |
| 55 | + | } | |
| 56 | + | ||
| 57 | + | // SetDemand records one live client subscription opening (active=true) or | |
| 58 | + | // closing (active=false). internal/usecase/broker/service.go calls it from | |
| 59 | + | // Service.Subscribe and serviceSubscription.Close, so the demand count tracks | |
| 60 | + | // exactly the connections a client is actively watching. The first | |
| 61 | + | // subscription requests one immediate observation per host; while demand | |
| 62 | + | // stays positive, the demand cadence schedules later observations. With no | |
| 63 | + | // subscription, only explicit RequestProbe calls dispatch. A disabled registry | |
| 64 | + | // still counts subscriptions but never observes. | |
| 65 | + | func (r *Registry) SetDemand(active bool) { | |
| 66 | + | r.mu.Lock() | |
| 67 | + | if active { | |
| 68 | + | if r.demand == 0 { | |
| 69 | + | r.localPending = r.local != nil | |
| 70 | + | for endpoint := range r.hosts { | |
| 71 | + | r.pending[endpoint] = true | |
| 72 | + | } | |
| 73 | + | } | |
| 74 | + | r.demand++ | |
| 75 | + | } else if r.demand > 0 { | |
| 76 | + | r.demand-- | |
| 77 | + | } | |
| 78 | + | r.mu.Unlock() | |
| 79 | + | r.hint() | |
| 80 | + | } | |
| 81 | + | ||
| 82 | + | // freshForLocked returns the demand cadence while subscribed. An absent | |
| 83 | + | // daemon uses the longer freshFor interval instead. Callers must hold r.mu. | |
| 84 | + | func (r *Registry) freshForLocked(local bool) time.Duration { | |
| 85 | + | if r.demand <= 0 { | |
| 86 | + | return r.freshFor | |
| 87 | + | } | |
| 88 | + | if local { | |
| 89 | + | return r.demandFreshForLocal | |
| 90 | + | } | |
| 91 | + | return r.demandFreshForRemote | |
| 92 | + | } | |
| 93 | + | ||
| 94 | + | func (r *Registry) hint() { | |
| 95 | + | select { | |
| 96 | + | case r.wake <- struct{}{}: | |
| 97 | + | default: | |
| 98 | + | } | |
| 99 | + | } | |
| 100 | + | ||
| 101 | + | // Run schedules observations until cancellation. Run owns the registry | |
| 102 | + | // lifecycle and may execute exactly once: a second call is refused so a | |
| 103 | + | // concurrent or restarted caller can never duplicate scheduling. On return it | |
| 104 | + | // clears in-flight observation state, publishes the settled projection, and | |
| 105 | + | // flushes durable persistence before any further write is refused. | |
| 106 | + | // | |
| 107 | + | // An observation-disabled registry runs the same lifecycle and drain but never | |
| 108 | + | // schedules observation: it arms no timer, starts no probe, and drops every | |
| 109 | + | // reconcile. It still owns and drains its durable writer, so a shutdown never | |
| 110 | + | // returns before the newest staged publication has reached the store. | |
| 111 | + | func (r *Registry) Run(ctx context.Context) { | |
| 112 | + | if !r.running.CompareAndSwap(runIdle, runActive) { | |
| 113 | + | r.log.Warn("broker: registry Run refused; single-run already started or finished") | |
| 114 | + | return | |
| 115 | + | } | |
| 116 | + | defer r.settle() | |
| 117 | + | if r.observationDisabled { | |
| 118 | + | <-ctx.Done() | |
| 119 | + | return | |
| 120 | + | } | |
| 121 | + | timer := r.clock.NewTimer(0) | |
| 122 | + | defer timer.Stop() | |
| 123 | + | results := make(chan probeResult, ports.BrokerMaxHosts) | |
| 124 | + | for { | |
| 125 | + | now := r.clock.Now() | |
| 126 | + | r.dispatch(ctx, now, results) | |
| 127 | + | timer.Reset(r.nextDelay(now)) | |
| 128 | + | select { | |
| 129 | + | case <-ctx.Done(): | |
| 130 | + | return | |
| 131 | + | case <-r.wake: | |
| 132 | + | case <-timer.C(): | |
| 133 | + | case result := <-results: | |
| 134 | + | if ctx.Err() != nil { | |
| 135 | + | // Shutdown wins: a cancellation is never recorded as an | |
| 136 | + | // observation failure. | |
| 137 | + | return | |
| 138 | + | } | |
| 139 | + | r.apply(result) | |
| 140 | + | } | |
| 141 | + | } | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | // settle is Run's cleanup: nothing stays in flight, the published projection | |
| 145 | + | // carries no transient Checking flag, durable persistence is flushed, and | |
| 146 | + | // every later Run call is refused. | |
| 147 | + | func (r *Registry) settle() { | |
| 148 | + | r.mu.Lock() | |
| 149 | + | r.cancelAllInflightLocked() | |
| 150 | + | if r.localAttempt != nil { | |
| 151 | + | r.localAttempt.cancel() | |
| 152 | + | r.localAttempt = nil | |
| 153 | + | } | |
| 154 | + | cleared := false | |
| 155 | + | if r.local != nil && r.localHost.Checking { | |
| 156 | + | r.localHost.Checking = false | |
| 157 | + | cleared = true | |
| 158 | + | } | |
| 159 | + | for endpoint, host := range r.hosts { | |
| 160 | + | if !host.Checking { | |
| 161 | + | continue | |
| 162 | + | } | |
| 163 | + | host.Checking = false | |
| 164 | + | r.hosts[endpoint] = host | |
| 165 | + | cleared = true | |
| 166 | + | } | |
| 167 | + | if cleared { | |
| 168 | + | r.publishLocked(true) | |
| 169 | + | } | |
| 170 | + | r.running.Store(runStopped) | |
| 171 | + | r.mu.Unlock() | |
| 172 | + | r.store.close() | |
| 173 | + | } | |
| 174 | + | ||
| 175 | + | // probeAttempt is one admitted observation: the registry-wide token that | |
| 176 | + | // fences its completion, and the cancel for the context derived from the run | |
| 177 | + | // context. Cancelling the derived context retires the attempt: a probe that | |
| 178 | + | // honors cancellation returns promptly, so a replaced or removed endpoint | |
| 179 | + | // never accumulates overlapping probes, and settle releases every attempt | |
| 180 | + | // before the run loop exits. | |
| 181 | + | type probeAttempt struct { | |
| 182 | + | token uint64 | |
| 183 | + | cancel context.CancelFunc | |
| 184 | + | } | |
| 185 | + | ||
| 186 | + | // probeResult is one completed observation bound to the exact attempt token | |
| 187 | + | // that started it. | |
| 188 | + | type probeResult struct { | |
| 189 | + | endpoint string | |
| 190 | + | registration domain.RemoteRegistration | |
| 191 | + | attempt uint64 | |
| 192 | + | snapshot ports.BrokerDaemonObservation | |
| 193 | + | err error | |
| 194 | + | at time.Time | |
| 195 | + | // local marks the result of one local probe attempt, which carries no | |
| 196 | + | // endpoint or registration and is applied to the configured local entry. | |
| 197 | + | local bool | |
| 198 | + | } | |
| 199 | + | ||
| 200 | + | func (r *Registry) dispatch(ctx context.Context, now time.Time, results chan<- probeResult) { | |
| 201 | + | r.mu.Lock() | |
| 202 | + | if r.demand == 0 && len(r.pending) == 0 && !r.localPending { | |
| 203 | + | r.mu.Unlock() | |
| 204 | + | return | |
| 205 | + | } | |
| 206 | + | started := false | |
| 207 | + | for endpoint, host := range r.hosts { | |
| 208 | + | if _, inflight := r.inflight[endpoint]; inflight { | |
| 209 | + | continue | |
| 210 | + | } | |
| 211 | + | explicit := r.pending[endpoint] | |
| 212 | + | if !explicit && (r.demand == 0 || !host.NextDue.IsZero() && now.Before(host.NextDue)) { | |
| 213 | + | continue | |
| 214 | + | } | |
| 215 | + | delete(r.pending, endpoint) | |
| 216 | + | r.attempts++ | |
| 217 | + | attempt := r.attempts | |
| 218 | + | probeCtx, cancel := context.WithCancel(ctx) | |
| 219 | + | r.inflight[endpoint] = &probeAttempt{token: attempt, cancel: cancel} | |
| 220 | + | if visibleCheck(host, explicit) { | |
| 221 | + | started = true | |
| 222 | + | host.Checking = true | |
| 223 | + | } | |
| 224 | + | host.LastAttempt = now | |
| 225 | + | r.hosts[endpoint] = host | |
| 226 | + | registration := host.Registration | |
| 227 | + | // The deadline is armed here, under r.mu and at dispatch time, so the | |
| 228 | + | // attempt's bound starts with the attempt rather than whenever its | |
| 229 | + | // goroutine is scheduled. | |
| 230 | + | deadline := r.clock.NewTimer(r.probeTimeout) | |
| 231 | + | go func(endpoint string, registration domain.RemoteRegistration, attempt uint64, probeCtx context.Context) { | |
| 232 | + | snapshot, err := observeBounded(probeCtx, deadline, func(ctx context.Context) (ports.BrokerDaemonObservation, error) { | |
| 233 | + | return r.probe.Probe(ctx, registration) | |
| 234 | + | }) | |
| 235 | + | result := probeResult{endpoint: endpoint, registration: registration, attempt: attempt, snapshot: snapshot, err: err, at: r.clock.Now()} | |
| 236 | + | select { | |
| 237 | + | case results <- result: | |
| 238 | + | case <-probeCtx.Done(): | |
| 239 | + | // The attempt was retired (replaced, removed, or shutdown), so | |
| 240 | + | // its completion is already fenced: dropping it keeps churn from | |
| 241 | + | // queueing results the run loop would only discard. | |
| 242 | + | } | |
| 243 | + | }(endpoint, registration, attempt, probeCtx) | |
| 244 | + | } | |
| 245 | + | // Only an observation that visibly started changes the projection; an | |
| 246 | + | // idle dispatch or a routine refresh must not bump the revision or wake | |
| 247 | + | // subscribers. The in-flight projection is never persisted: Checking is | |
| 248 | + | // transient. | |
| 249 | + | if r.dispatchLocalLocked(ctx, now, results) { | |
| 250 | + | started = true | |
| 251 | + | } | |
| 252 | + | if started { | |
| 253 | + | r.publishLocked(false) | |
| 254 | + | } | |
| 255 | + | r.mu.Unlock() | |
| 256 | + | } | |
| 257 | + | ||
| 258 | + | // visibleCheck reports whether starting an observation publishes the transient | |
| 259 | + | // Checking state. A routine scheduled refresh of a reachable daemon stays | |
| 260 | + | // silent until its result lands, so an idle watched broker publishes once per | |
| 261 | + | // observation instead of twice; an explicit request or a daemon that is not | |
| 262 | + | // confirmed reachable still shows that it is being checked. | |
| 263 | + | func visibleCheck(host ports.BrokerDaemonObservation, explicit bool) bool { | |
| 264 | + | return explicit || host.Availability != domain.RemoteAvailabilityReachable | |
| 265 | + | } | |
| 266 | + | ||
| 267 | + | // observeBounded runs one observation under its attempt deadline. retire is the | |
| 268 | + | // attempt's retirement context: a replaced, removed, or settled attempt returns | |
| 269 | + | // at once and its completion is fenced by the caller. The probe itself runs on | |
| 270 | + | // a context derived from retire that the deadline also cancels, so a | |
| 271 | + | // ctx-honoring probe unwinds its dial, handshake, or exchange when either | |
| 272 | + | // fires. The attempt never waits for the probe past its deadline: a probe that | |
| 273 | + | // ignores cancellation is abandoned (its late result is discarded) and the | |
| 274 | + | // attempt still reports errProbeTimeout, so the endpoint's single in-flight | |
| 275 | + | // slot is always released. | |
| 276 | + | func observeBounded(retire context.Context, deadline ports.Timer, observe func(context.Context) (ports.BrokerDaemonObservation, error)) (ports.BrokerDaemonObservation, error) { | |
| 277 | + | defer deadline.Stop() | |
| 278 | + | probeCtx, cancel := context.WithCancelCause(retire) | |
| 279 | + | defer cancel(nil) | |
| 280 | + | type outcome struct { | |
| 281 | + | snapshot ports.BrokerDaemonObservation | |
| 282 | + | err error | |
| 283 | + | } | |
| 284 | + | done := make(chan outcome, 1) | |
| 285 | + | go func() { | |
| 286 | + | snapshot, err := observe(probeCtx) | |
| 287 | + | done <- outcome{snapshot: snapshot, err: err} | |
| 288 | + | }() | |
| 289 | + | select { | |
| 290 | + | case result := <-done: | |
| 291 | + | return result.snapshot, result.err | |
| 292 | + | case <-deadline.C(): | |
| 293 | + | cancel(errProbeTimeout) | |
| 294 | + | return ports.BrokerDaemonObservation{}, errProbeTimeout | |
| 295 | + | case <-retire.Done(): | |
| 296 | + | return ports.BrokerDaemonObservation{}, retire.Err() | |
| 297 | + | } | |
| 298 | + | } | |
| 299 | + | ||
| 300 | + | func (r *Registry) apply(result probeResult) { | |
| 301 | + | r.mu.Lock() | |
| 302 | + | defer r.mu.Unlock() | |
| 303 | + | if result.local { | |
| 304 | + | r.applyLocal(result) | |
| 305 | + | return | |
| 306 | + | } | |
| 307 | + | current, ok := r.hosts[result.endpoint] | |
| 308 | + | attempt := r.inflight[result.endpoint] | |
| 309 | + | if !ok || !current.Registration.Equal(result.registration) || attempt == nil || attempt.token != result.attempt { | |
| 310 | + | // A stale completion touches neither current authority nor the | |
| 311 | + | // in-flight slot, which may already belong to a newer registration or | |
| 312 | + | // attempt. | |
| 313 | + | return | |
| 314 | + | } | |
| 315 | + | delete(r.inflight, result.endpoint) | |
| 316 | + | // The probe has returned, so releasing its derived context here only | |
| 317 | + | // frees the attempt's resources; the record is gone before any new | |
| 318 | + | // attempt can start. | |
| 319 | + | attempt.cancel() | |
| 320 | + | current.Checking = false | |
| 321 | + | current.LastAttempt = result.at | |
| 322 | + | ||
| 323 | + | kind, availability, failed := observationOutcome(result) | |
| 324 | + | if !failed && availability == domain.RemoteAvailabilityNoDaemon { | |
| 325 | + | current.Availability = availability | |
| 326 | + | current.LastSuccess = result.at | |
| 327 | + | // An absent daemon is stable: demand does not require a fresh SSH | |
| 328 | + | // handshake every two seconds just to confirm it is still absent. | |
| 329 | + | current.NextDue = result.at.Add(r.jitter(r.freshFor, result.endpoint, result.attempt)) | |
| 330 | + | current.ConsecutiveFailures = 0 | |
| 331 | + | current.LastFailure = domain.RemoteFailure{} | |
| 332 | + | current.Identity = "" | |
| 333 | + | current.Incarnation = ports.BrokerDaemonIncarnation{} | |
| 334 | + | current.ProtocolVersion = 0 | |
| 335 | + | current.Capabilities = 0 | |
| 336 | + | current.InventoryKnown = false | |
| 337 | + | current.Sessions = nil | |
| 338 | + | r.hosts[result.endpoint] = current | |
| 339 | + | r.publishLocked(true) | |
| 340 | + | return | |
| 341 | + | } | |
| 342 | + | if !failed { | |
| 343 | + | // Configured authority comes only from the registry's own projection: | |
| 344 | + | // a probe result supplies observed state, never endpoint, registration, | |
| 345 | + | // policy, display origin, or rank. | |
| 346 | + | observed := stampObservationAuthority(result.snapshot.Clone(), current) | |
| 347 | + | observed.Checking = false | |
| 348 | + | observed.LastAttempt = result.at | |
| 349 | + | observed.LastSuccess = result.at | |
| 350 | + | observed.NextDue = result.at.Add(r.jitter(r.freshForLocked(false), result.endpoint, result.attempt)) | |
| 351 | + | observed.ConsecutiveFailures = 0 | |
| 352 | + | observed.LastFailure = domain.RemoteFailure{} | |
| 353 | + | observed.FailureEpisode = current.FailureEpisode | |
| 354 | + | // The published observation must be fully valid once authority is | |
| 355 | + | // stamped: a probe that invents partial identity (identity without a | |
| 356 | + | // protocol version) or an inventory without a known catalogue is an | |
| 357 | + | // invalid response, never a published observation. The durable rule is | |
| 358 | + | // re-checked too: a projection the offline store would reject is never | |
| 359 | + | // published as if it had persisted. The store applies the same rule. | |
| 360 | + | if err := observed.Validate(); err != nil { | |
| 361 | + | r.log.Debug("broker: observation is invalid", "endpoint", result.endpoint, "err", err) | |
| 362 | + | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |
| 363 | + | } else if err := ports.ValidateDurableHostProjection(observed); err != nil { | |
| 364 | + | r.log.Debug("broker: observation is not persistable", "endpoint", result.endpoint, "err", err) | |
| 365 | + | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |
| 366 | + | } else { | |
| 367 | + | r.hosts[result.endpoint] = observed | |
| 368 | + | r.publishLocked(true) | |
| 369 | + | return | |
| 370 | + | } | |
| 371 | + | } | |
| 372 | + | // Every non-reachable outcome is a typed failure on the capped retry | |
| 373 | + | // cadence: the last success, last-known inventory, and failure episode | |
| 374 | + | // counter stay authoritative. | |
| 375 | + | current.Availability = availability | |
| 376 | + | if current.ConsecutiveFailures == 0 { | |
| 377 | + | current.FailureEpisode++ | |
| 378 | + | } | |
| 379 | + | current.ConsecutiveFailures++ | |
| 380 | + | current.LastFailure = domain.RemoteFailure{Kind: kind, Err: result.err} | |
| 381 | + | delay := r.retryDelay(current.ConsecutiveFailures) | |
| 382 | + | if r.demand > 0 { | |
| 383 | + | delay = demandRetryDelay(r.demandFreshForRemote, current.ConsecutiveFailures) | |
| 384 | + | } | |
| 385 | + | current.NextDue = result.at.Add(r.jitter(delay, result.endpoint, result.attempt)) | |
| 386 | + | r.log.Debug("broker_probe_failed", "endpoint", result.endpoint, "kind", kind, "failures", current.ConsecutiveFailures, "err", result.err, "cause", errors.Unwrap(result.err)) | |
| 387 | + | r.hosts[result.endpoint] = current | |
| 388 | + | r.publishLocked(true) | |
| 389 | + | } | |
| 390 | + | ||
| 391 | + | // observationOutcome classifies a completed observation. It reports | |
| 392 | + | // failed=false only for a confirmed reachable result; every other outcome | |
| 393 | + | // carries the typed failure kind and the availability to publish. The caller | |
| 394 | + | // additionally checks the adopted projection against the durable rules | |
| 395 | + | // (ports.ValidateDurableHostProjection), which cover the session bound and | |
| 396 | + | // catalogue validity for the exact incarnation and success time. | |
| 397 | + | func observationOutcome(result probeResult) (domain.RemoteFailureKind, domain.RemoteAvailability, bool) { | |
| 398 | + | if result.err != nil { | |
| 399 | + | kind := domain.RemoteFailureTransport | |
| 400 | + | var typed domain.RemoteFailure | |
| 401 | + | var brokerErr ports.BrokerError | |
| 402 | + | switch { | |
| 403 | + | case errors.As(result.err, &typed) && typed.Kind != domain.RemoteFailureNone: | |
| 404 | + | // The dialer classified the failure (for example SSH authentication). | |
| 405 | + | kind = typed.Kind | |
| 406 | + | case errors.As(result.err, &brokerErr) && (brokerErr.Code == ports.BrokerErrorConflictingPolicy || brokerErr.Code == ports.BrokerErrorIncompatible): | |
| 407 | + | // SSH worked; the daemon's identity or version was refused. | |
| 408 | + | kind = domain.RemoteFailureIncompatible | |
| 409 | + | case errors.Is(result.err, context.DeadlineExceeded): | |
| 410 | + | kind = domain.RemoteFailureTimeout | |
| 411 | + | } | |
| 412 | + | return kind, availabilityFor(kind), true | |
| 413 | + | } | |
| 414 | + | switch result.snapshot.Availability { | |
| 415 | + | case domain.RemoteAvailabilityReachable: | |
| 416 | + | return domain.RemoteFailureNone, domain.RemoteAvailabilityReachable, false | |
| 417 | + | case domain.RemoteAvailabilityNoDaemon: | |
| 418 | + | return domain.RemoteFailureNone, domain.RemoteAvailabilityNoDaemon, false | |
| 419 | + | case domain.RemoteAvailabilityIncompatible: | |
| 420 | + | return domain.RemoteFailureIncompatible, domain.RemoteAvailabilityIncompatible, true | |
| 421 | + | case domain.RemoteAvailabilityAuthFailed: | |
| 422 | + | return domain.RemoteFailureAuthentication, domain.RemoteAvailabilityAuthFailed, true | |
| 423 | + | case domain.RemoteAvailabilityInvalidResponse: | |
| 424 | + | return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |
| 425 | + | default: | |
| 426 | + | // Unknown or Unreachable without an error still means reachability was | |
| 427 | + | // not confirmed, so it retries on the typed backoff cadence. | |
| 428 | + | return domain.RemoteFailureTransport, domain.RemoteAvailabilityUnreachable, true | |
| 429 | + | } | |
| 430 | + | } | |
| 431 | + | ||
| 432 | + | // availabilityFor maps a typed failure kind to the published availability. | |
| 433 | + | // The typed kind is preserved separately for notice policy. | |
| 434 | + | func availabilityFor(kind domain.RemoteFailureKind) domain.RemoteAvailability { | |
| 435 | + | switch kind { | |
| 436 | + | case domain.RemoteFailureAuthentication: | |
| 437 | + | return domain.RemoteAvailabilityAuthFailed | |
| 438 | + | case domain.RemoteFailureIncompatible: | |
| 439 | + | return domain.RemoteAvailabilityIncompatible | |
| 440 | + | case domain.RemoteFailureInvalidResponse: | |
| 441 | + | return domain.RemoteAvailabilityInvalidResponse | |
| 442 | + | default: | |
| 443 | + | return domain.RemoteAvailabilityUnreachable | |
| 444 | + | } | |
| 445 | + | } | |
| 446 | + | ||
| 447 | + | // demandRetryDelay is the watched-host failure backoff: the demand cadence | |
| 448 | + | // doubled per consecutive failure, capped at demandRetryMax. | |
| 449 | + | func demandRetryDelay(base time.Duration, failures uint) time.Duration { | |
| 450 | + | delay := base | |
| 451 | + | for i := uint(1); i < failures && delay < demandRetryMax; i++ { | |
| 452 | + | delay *= 2 | |
| 453 | + | } | |
| 454 | + | return min(delay, max(base, demandRetryMax)) | |
| 455 | + | } | |
| 456 | + | ||
| 457 | + | // retryDelay is the capped exponential retry cadence for a failure streak. | |
| 458 | + | func (r *Registry) retryDelay(failures uint) time.Duration { | |
| 459 | + | delay := r.retryBase | |
| 460 | + | for i := uint(1); i < failures; i++ { | |
| 461 | + | if delay >= r.retryMax { | |
| 462 | + | return r.retryMax | |
| 463 | + | } | |
| 464 | + | delay *= 2 | |
| 465 | + | } | |
| 466 | + | if delay > r.retryMax { | |
| 467 | + | return r.retryMax | |
| 468 | + | } | |
| 469 | + | return delay | |
| 470 | + | } | |
| 471 | + | ||
| 472 | + | // jittered spreads a base interval deterministically across endpoint and | |
| 473 | + | // attempt: identical inputs always yield the identical duration, and the | |
| 474 | + | // result stays within 90%-110% of the base. No wall clock or global | |
| 475 | + | // randomness participates, so retry and healthy-refresh schedules are | |
| 476 | + | // reproducible instead of racing a shared PRNG. | |
| 477 | + | func jittered(base time.Duration, endpoint string, attempt uint64) time.Duration { | |
| 478 | + | if base <= 0 { | |
| 479 | + | return base | |
| 480 | + | } | |
| 481 | + | h := fnv.New32a() | |
| 482 | + | h.Write([]byte(endpoint)) | |
| 483 | + | var counter [8]byte | |
| 484 | + | binary.BigEndian.PutUint64(counter[:], attempt) | |
| 485 | + | h.Write(counter[:]) | |
| 486 | + | spread := int64(h.Sum32() % 21) | |
| 487 | + | return base*90/100 + time.Duration(spread)*base/100 | |
| 488 | + | } | |
| 489 | + | ||
| 490 | + | func (r *Registry) nextDelay(now time.Time) time.Duration { | |
| 491 | + | r.mu.Lock() | |
| 492 | + | defer r.mu.Unlock() | |
| 493 | + | if r.demand == 0 { | |
| 494 | + | if r.localPending || len(r.pending) > 0 { | |
| 495 | + | return 0 | |
| 496 | + | } | |
| 497 | + | return time.Hour | |
| 498 | + | } | |
| 499 | + | var earliest time.Time | |
| 500 | + | if r.local != nil && r.localAttempt == nil { | |
| 501 | + | if r.localHost.NextDue.IsZero() { | |
| 502 | + | return 0 | |
| 503 | + | } | |
| 504 | + | if earliest.IsZero() || r.localHost.NextDue.Before(earliest) { | |
| 505 | + | earliest = r.localHost.NextDue | |
| 506 | + | } | |
| 507 | + | } | |
| 508 | + | for endpoint, host := range r.hosts { | |
| 509 | + | // An observation already in flight is completed by its result, never | |
| 510 | + | // by the schedule; waiting on it here would spin the run loop. | |
| 511 | + | if _, inflight := r.inflight[endpoint]; inflight { | |
| 512 | + | continue | |
| 513 | + | } | |
| 514 | + | if r.pending[endpoint] || host.NextDue.IsZero() { | |
| 515 | + | return 0 | |
| 516 | + | } | |
| 517 | + | if earliest.IsZero() || host.NextDue.Before(earliest) { | |
| 518 | + | earliest = host.NextDue | |
| 519 | + | } | |
| 520 | + | } | |
| 521 | + | if earliest.IsZero() { | |
| 522 | + | return time.Hour | |
| 523 | + | } | |
| 524 | + | if delay := earliest.Sub(now); delay > 0 { | |
| 525 | + | return delay | |
| 526 | + | } | |
| 527 | + | return 0 | |
| 528 | + | } | |
| @@ -0,0 +1,528 @@ | |||||
| 1 | + | package broker | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "context" | |||
| 5 | + | "encoding/binary" | |||
| 6 | + | "errors" | |||
| 7 | + | "hash/fnv" | |||
| 8 | + | "time" | |||
| 9 | + | ||||
| 10 | + | "github.com/bnema/vev/internal/domain" | |||
| 11 | + | "github.com/bnema/vev/internal/ports" | |||
| 12 | + | ) | |||
| 13 | + | ||||
| 14 | + | // cancelInflightLocked cancels the in-flight attempt for one endpoint and | |||
| 15 | + | // drops its record. Callers must hold r.mu: the retired attempt's context is | |||
| 16 | + | // cancelled before the record disappears, so no replacement attempt can start | |||
| 17 | + | // while the retired probe's context is still live. | |||
| 18 | + | func (r *Registry) cancelInflightLocked(endpoint string) { | |||
| 19 | + | if attempt := r.inflight[endpoint]; attempt != nil { | |||
| 20 | + | attempt.cancel() | |||
| 21 | + | delete(r.inflight, endpoint) | |||
| 22 | + | } | |||
| 23 | + | } | |||
| 24 | + | ||||
| 25 | + | // cancelAllInflightLocked cancels every in-flight attempt and drops the | |||
| 26 | + | // records. Callers must hold r.mu. | |||
| 27 | + | func (r *Registry) cancelAllInflightLocked() { | |||
| 28 | + | for endpoint, attempt := range r.inflight { | |||
| 29 | + | attempt.cancel() | |||
| 30 | + | delete(r.inflight, endpoint) | |||
| 31 | + | } | |||
| 32 | + | } | |||
| 33 | + | ||||
| 34 | + | // RequestProbe is non-blocking and coalesces concurrent demand per endpoint. | |||
| 35 | + | // A disabled registry never observes, so the demand is dropped without | |||
| 36 | + | // scheduling anything. An empty endpoint requests a local re-probe instead of a | |||
| 37 | + | // configured remote: the broker's own machine daemon is re-observed ahead of | |||
| 38 | + | // its scheduled freshness window, honored by dispatchLocalLocked exactly like | |||
| 39 | + | // pending honors remote demand. A registry with no local observation producer | |||
| 40 | + | // drops it, just like an unknown remote endpoint is dropped. | |||
| 41 | + | func (r *Registry) RequestProbe(endpoint string) { | |||
| 42 | + | if r.observationDisabled { | |||
| 43 | + | return | |||
| 44 | + | } | |||
| 45 | + | r.mu.Lock() | |||
| 46 | + | if endpoint == "" { | |||
| 47 | + | if r.local != nil { | |||
| 48 | + | r.localPending = true | |||
| 49 | + | } | |||
| 50 | + | } else if _, ok := r.hosts[endpoint]; ok { | |||
| 51 | + | r.pending[endpoint] = true | |||
| 52 | + | } | |||
| 53 | + | r.mu.Unlock() | |||
| 54 | + | r.hint() | |||
| 55 | + | } | |||
| 56 | + | ||||
| 57 | + | // SetDemand records one live client subscription opening (active=true) or | |||
| 58 | + | // closing (active=false). internal/usecase/broker/service.go calls it from | |||
| 59 | + | // Service.Subscribe and serviceSubscription.Close, so the demand count tracks | |||
| 60 | + | // exactly the connections a client is actively watching. The first | |||
| 61 | + | // subscription requests one immediate observation per host; while demand | |||
| 62 | + | // stays positive, the demand cadence schedules later observations. With no | |||
| 63 | + | // subscription, only explicit RequestProbe calls dispatch. A disabled registry | |||
| 64 | + | // still counts subscriptions but never observes. | |||
| 65 | + | func (r *Registry) SetDemand(active bool) { | |||
| 66 | + | r.mu.Lock() | |||
| 67 | + | if active { | |||
| 68 | + | if r.demand == 0 { | |||
| 69 | + | r.localPending = r.local != nil | |||
| 70 | + | for endpoint := range r.hosts { | |||
| 71 | + | r.pending[endpoint] = true | |||
| 72 | + | } | |||
| 73 | + | } | |||
| 74 | + | r.demand++ | |||
| 75 | + | } else if r.demand > 0 { | |||
| 76 | + | r.demand-- | |||
| 77 | + | } | |||
| 78 | + | r.mu.Unlock() | |||
| 79 | + | r.hint() | |||
| 80 | + | } | |||
| 81 | + | ||||
| 82 | + | // freshForLocked returns the demand cadence while subscribed. An absent | |||
| 83 | + | // daemon uses the longer freshFor interval instead. Callers must hold r.mu. | |||
| 84 | + | func (r *Registry) freshForLocked(local bool) time.Duration { | |||
| 85 | + | if r.demand <= 0 { | |||
| 86 | + | return r.freshFor | |||
| 87 | + | } | |||
| 88 | + | if local { | |||
| 89 | + | return r.demandFreshForLocal | |||
| 90 | + | } | |||
| 91 | + | return r.demandFreshForRemote | |||
| 92 | + | } | |||
| 93 | + | ||||
| 94 | + | func (r *Registry) hint() { | |||
| 95 | + | select { | |||
| 96 | + | case r.wake <- struct{}{}: | |||
| 97 | + | default: | |||
| 98 | + | } | |||
| 99 | + | } | |||
| 100 | + | ||||
| 101 | + | // Run schedules observations until cancellation. Run owns the registry | |||
| 102 | + | // lifecycle and may execute exactly once: a second call is refused so a | |||
| 103 | + | // concurrent or restarted caller can never duplicate scheduling. On return it | |||
| 104 | + | // clears in-flight observation state, publishes the settled projection, and | |||
| 105 | + | // flushes durable persistence before any further write is refused. | |||
| 106 | + | // | |||
| 107 | + | // An observation-disabled registry runs the same lifecycle and drain but never | |||
| 108 | + | // schedules observation: it arms no timer, starts no probe, and drops every | |||
| 109 | + | // reconcile. It still owns and drains its durable writer, so a shutdown never | |||
| 110 | + | // returns before the newest staged publication has reached the store. | |||
| 111 | + | func (r *Registry) Run(ctx context.Context) { | |||
| 112 | + | if !r.running.CompareAndSwap(runIdle, runActive) { | |||
| 113 | + | r.log.Warn("broker: registry Run refused; single-run already started or finished") | |||
| 114 | + | return | |||
| 115 | + | } | |||
| 116 | + | defer r.settle() | |||
| 117 | + | if r.observationDisabled { | |||
| 118 | + | <-ctx.Done() | |||
| 119 | + | return | |||
| 120 | + | } | |||
| 121 | + | timer := r.clock.NewTimer(0) | |||
| 122 | + | defer timer.Stop() | |||
| 123 | + | results := make(chan probeResult, ports.BrokerMaxHosts) | |||
| 124 | + | for { | |||
| 125 | + | now := r.clock.Now() | |||
| 126 | + | r.dispatch(ctx, now, results) | |||
| 127 | + | timer.Reset(r.nextDelay(now)) | |||
| 128 | + | select { | |||
| 129 | + | case <-ctx.Done(): | |||
| 130 | + | return | |||
| 131 | + | case <-r.wake: | |||
| 132 | + | case <-timer.C(): | |||
| 133 | + | case result := <-results: | |||
| 134 | + | if ctx.Err() != nil { | |||
| 135 | + | // Shutdown wins: a cancellation is never recorded as an | |||
| 136 | + | // observation failure. | |||
| 137 | + | return | |||
| 138 | + | } | |||
| 139 | + | r.apply(result) | |||
| 140 | + | } | |||
| 141 | + | } | |||
| 142 | + | } | |||
| 143 | + | ||||
| 144 | + | // settle is Run's cleanup: nothing stays in flight, the published projection | |||
| 145 | + | // carries no transient Checking flag, durable persistence is flushed, and | |||
| 146 | + | // every later Run call is refused. | |||
| 147 | + | func (r *Registry) settle() { | |||
| 148 | + | r.mu.Lock() | |||
| 149 | + | r.cancelAllInflightLocked() | |||
| 150 | + | if r.localAttempt != nil { | |||
| 151 | + | r.localAttempt.cancel() | |||
| 152 | + | r.localAttempt = nil | |||
| 153 | + | } | |||
| 154 | + | cleared := false | |||
| 155 | + | if r.local != nil && r.localHost.Checking { | |||
| 156 | + | r.localHost.Checking = false | |||
| 157 | + | cleared = true | |||
| 158 | + | } | |||
| 159 | + | for endpoint, host := range r.hosts { | |||
| 160 | + | if !host.Checking { | |||
| 161 | + | continue | |||
| 162 | + | } | |||
| 163 | + | host.Checking = false | |||
| 164 | + | r.hosts[endpoint] = host | |||
| 165 | + | cleared = true | |||
| 166 | + | } | |||
| 167 | + | if cleared { | |||
| 168 | + | r.publishLocked(true) | |||
| 169 | + | } | |||
| 170 | + | r.running.Store(runStopped) | |||
| 171 | + | r.mu.Unlock() | |||
| 172 | + | r.store.close() | |||
| 173 | + | } | |||
| 174 | + | ||||
| 175 | + | // probeAttempt is one admitted observation: the registry-wide token that | |||
| 176 | + | // fences its completion, and the cancel for the context derived from the run | |||
| 177 | + | // context. Cancelling the derived context retires the attempt: a probe that | |||
| 178 | + | // honors cancellation returns promptly, so a replaced or removed endpoint | |||
| 179 | + | // never accumulates overlapping probes, and settle releases every attempt | |||
| 180 | + | // before the run loop exits. | |||
| 181 | + | type probeAttempt struct { | |||
| 182 | + | token uint64 | |||
| 183 | + | cancel context.CancelFunc | |||
| 184 | + | } | |||
| 185 | + | ||||
| 186 | + | // probeResult is one completed observation bound to the exact attempt token | |||
| 187 | + | // that started it. | |||
| 188 | + | type probeResult struct { | |||
| 189 | + | endpoint string | |||
| 190 | + | registration domain.RemoteRegistration | |||
| 191 | + | attempt uint64 | |||
| 192 | + | snapshot ports.BrokerDaemonObservation | |||
| 193 | + | err error | |||
| 194 | + | at time.Time | |||
| 195 | + | // local marks the result of one local probe attempt, which carries no | |||
| 196 | + | // endpoint or registration and is applied to the configured local entry. | |||
| 197 | + | local bool | |||
| 198 | + | } | |||
| 199 | + | ||||
| 200 | + | func (r *Registry) dispatch(ctx context.Context, now time.Time, results chan<- probeResult) { | |||
| 201 | + | r.mu.Lock() | |||
| 202 | + | if r.demand == 0 && len(r.pending) == 0 && !r.localPending { | |||
| 203 | + | r.mu.Unlock() | |||
| 204 | + | return | |||
| 205 | + | } | |||
| 206 | + | started := false | |||
| 207 | + | for endpoint, host := range r.hosts { | |||
| 208 | + | if _, inflight := r.inflight[endpoint]; inflight { | |||
| 209 | + | continue | |||
| 210 | + | } | |||
| 211 | + | explicit := r.pending[endpoint] | |||
| 212 | + | if !explicit && (r.demand == 0 || !host.NextDue.IsZero() && now.Before(host.NextDue)) { | |||
| 213 | + | continue | |||
| 214 | + | } | |||
| 215 | + | delete(r.pending, endpoint) | |||
| 216 | + | r.attempts++ | |||
| 217 | + | attempt := r.attempts | |||
| 218 | + | probeCtx, cancel := context.WithCancel(ctx) | |||
| 219 | + | r.inflight[endpoint] = &probeAttempt{token: attempt, cancel: cancel} | |||
| 220 | + | if visibleCheck(host, explicit) { | |||
| 221 | + | started = true | |||
| 222 | + | host.Checking = true | |||
| 223 | + | } | |||
| 224 | + | host.LastAttempt = now | |||
| 225 | + | r.hosts[endpoint] = host | |||
| 226 | + | registration := host.Registration | |||
| 227 | + | // The deadline is armed here, under r.mu and at dispatch time, so the | |||
| 228 | + | // attempt's bound starts with the attempt rather than whenever its | |||
| 229 | + | // goroutine is scheduled. | |||
| 230 | + | deadline := r.clock.NewTimer(r.probeTimeout) | |||
| 231 | + | go func(endpoint string, registration domain.RemoteRegistration, attempt uint64, probeCtx context.Context) { | |||
| 232 | + | snapshot, err := observeBounded(probeCtx, deadline, func(ctx context.Context) (ports.BrokerDaemonObservation, error) { | |||
| 233 | + | return r.probe.Probe(ctx, registration) | |||
| 234 | + | }) | |||
| 235 | + | result := probeResult{endpoint: endpoint, registration: registration, attempt: attempt, snapshot: snapshot, err: err, at: r.clock.Now()} | |||
| 236 | + | select { | |||
| 237 | + | case results <- result: | |||
| 238 | + | case <-probeCtx.Done(): | |||
| 239 | + | // The attempt was retired (replaced, removed, or shutdown), so | |||
| 240 | + | // its completion is already fenced: dropping it keeps churn from | |||
| 241 | + | // queueing results the run loop would only discard. | |||
| 242 | + | } | |||
| 243 | + | }(endpoint, registration, attempt, probeCtx) | |||
| 244 | + | } | |||
| 245 | + | // Only an observation that visibly started changes the projection; an | |||
| 246 | + | // idle dispatch or a routine refresh must not bump the revision or wake | |||
| 247 | + | // subscribers. The in-flight projection is never persisted: Checking is | |||
| 248 | + | // transient. | |||
| 249 | + | if r.dispatchLocalLocked(ctx, now, results) { | |||
| 250 | + | started = true | |||
| 251 | + | } | |||
| 252 | + | if started { | |||
| 253 | + | r.publishLocked(false) | |||
| 254 | + | } | |||
| 255 | + | r.mu.Unlock() | |||
| 256 | + | } | |||
| 257 | + | ||||
| 258 | + | // visibleCheck reports whether starting an observation publishes the transient | |||
| 259 | + | // Checking state. A routine scheduled refresh of a reachable daemon stays | |||
| 260 | + | // silent until its result lands, so an idle watched broker publishes once per | |||
| 261 | + | // observation instead of twice; an explicit request or a daemon that is not | |||
| 262 | + | // confirmed reachable still shows that it is being checked. | |||
| 263 | + | func visibleCheck(host ports.BrokerDaemonObservation, explicit bool) bool { | |||
| 264 | + | return explicit || host.Availability != domain.RemoteAvailabilityReachable | |||
| 265 | + | } | |||
| 266 | + | ||||
| 267 | + | // observeBounded runs one observation under its attempt deadline. retire is the | |||
| 268 | + | // attempt's retirement context: a replaced, removed, or settled attempt returns | |||
| 269 | + | // at once and its completion is fenced by the caller. The probe itself runs on | |||
| 270 | + | // a context derived from retire that the deadline also cancels, so a | |||
| 271 | + | // ctx-honoring probe unwinds its dial, handshake, or exchange when either | |||
| 272 | + | // fires. The attempt never waits for the probe past its deadline: a probe that | |||
| 273 | + | // ignores cancellation is abandoned (its late result is discarded) and the | |||
| 274 | + | // attempt still reports errProbeTimeout, so the endpoint's single in-flight | |||
| 275 | + | // slot is always released. | |||
| 276 | + | func observeBounded(retire context.Context, deadline ports.Timer, observe func(context.Context) (ports.BrokerDaemonObservation, error)) (ports.BrokerDaemonObservation, error) { | |||
| 277 | + | defer deadline.Stop() | |||
| 278 | + | probeCtx, cancel := context.WithCancelCause(retire) | |||
| 279 | + | defer cancel(nil) | |||
| 280 | + | type outcome struct { | |||
| 281 | + | snapshot ports.BrokerDaemonObservation | |||
| 282 | + | err error | |||
| 283 | + | } | |||
| 284 | + | done := make(chan outcome, 1) | |||
| 285 | + | go func() { | |||
| 286 | + | snapshot, err := observe(probeCtx) | |||
| 287 | + | done <- outcome{snapshot: snapshot, err: err} | |||
| 288 | + | }() | |||
| 289 | + | select { | |||
| 290 | + | case result := <-done: | |||
| 291 | + | return result.snapshot, result.err | |||
| 292 | + | case <-deadline.C(): | |||
| 293 | + | cancel(errProbeTimeout) | |||
| 294 | + | return ports.BrokerDaemonObservation{}, errProbeTimeout | |||
| 295 | + | case <-retire.Done(): | |||
| 296 | + | return ports.BrokerDaemonObservation{}, retire.Err() | |||
| 297 | + | } | |||
| 298 | + | } | |||
| 299 | + | ||||
| 300 | + | func (r *Registry) apply(result probeResult) { | |||
| 301 | + | r.mu.Lock() | |||
| 302 | + | defer r.mu.Unlock() | |||
| 303 | + | if result.local { | |||
| 304 | + | r.applyLocal(result) | |||
| 305 | + | return | |||
| 306 | + | } | |||
| 307 | + | current, ok := r.hosts[result.endpoint] | |||
| 308 | + | attempt := r.inflight[result.endpoint] | |||
| 309 | + | if !ok || !current.Registration.Equal(result.registration) || attempt == nil || attempt.token != result.attempt { | |||
| 310 | + | // A stale completion touches neither current authority nor the | |||
| 311 | + | // in-flight slot, which may already belong to a newer registration or | |||
| 312 | + | // attempt. | |||
| 313 | + | return | |||
| 314 | + | } | |||
| 315 | + | delete(r.inflight, result.endpoint) | |||
| 316 | + | // The probe has returned, so releasing its derived context here only | |||
| 317 | + | // frees the attempt's resources; the record is gone before any new | |||
| 318 | + | // attempt can start. | |||
| 319 | + | attempt.cancel() | |||
| 320 | + | current.Checking = false | |||
| 321 | + | current.LastAttempt = result.at | |||
| 322 | + | ||||
| 323 | + | kind, availability, failed := observationOutcome(result) | |||
| 324 | + | if !failed && availability == domain.RemoteAvailabilityNoDaemon { | |||
| 325 | + | current.Availability = availability | |||
| 326 | + | current.LastSuccess = result.at | |||
| 327 | + | // An absent daemon is stable: demand does not require a fresh SSH | |||
| 328 | + | // handshake every two seconds just to confirm it is still absent. | |||
| 329 | + | current.NextDue = result.at.Add(r.jitter(r.freshFor, result.endpoint, result.attempt)) | |||
| 330 | + | current.ConsecutiveFailures = 0 | |||
| 331 | + | current.LastFailure = domain.RemoteFailure{} | |||
| 332 | + | current.Identity = "" | |||
| 333 | + | current.Incarnation = ports.BrokerDaemonIncarnation{} | |||
| 334 | + | current.ProtocolVersion = 0 | |||
| 335 | + | current.Capabilities = 0 | |||
| 336 | + | current.InventoryKnown = false | |||
| 337 | + | current.Sessions = nil | |||
| 338 | + | r.hosts[result.endpoint] = current | |||
| 339 | + | r.publishLocked(true) | |||
| 340 | + | return | |||
| 341 | + | } | |||
| 342 | + | if !failed { | |||
| 343 | + | // Configured authority comes only from the registry's own projection: | |||
| 344 | + | // a probe result supplies observed state, never endpoint, registration, | |||
| 345 | + | // policy, display origin, or rank. | |||
| 346 | + | observed := stampObservationAuthority(result.snapshot.Clone(), current) | |||
| 347 | + | observed.Checking = false | |||
| 348 | + | observed.LastAttempt = result.at | |||
| 349 | + | observed.LastSuccess = result.at | |||
| 350 | + | observed.NextDue = result.at.Add(r.jitter(r.freshForLocked(false), result.endpoint, result.attempt)) | |||
| 351 | + | observed.ConsecutiveFailures = 0 | |||
| 352 | + | observed.LastFailure = domain.RemoteFailure{} | |||
| 353 | + | observed.FailureEpisode = current.FailureEpisode | |||
| 354 | + | // The published observation must be fully valid once authority is | |||
| 355 | + | // stamped: a probe that invents partial identity (identity without a | |||
| 356 | + | // protocol version) or an inventory without a known catalogue is an | |||
| 357 | + | // invalid response, never a published observation. The durable rule is | |||
| 358 | + | // re-checked too: a projection the offline store would reject is never | |||
| 359 | + | // published as if it had persisted. The store applies the same rule. | |||
| 360 | + | if err := observed.Validate(); err != nil { | |||
| 361 | + | r.log.Debug("broker: observation is invalid", "endpoint", result.endpoint, "err", err) | |||
| 362 | + | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |||
| 363 | + | } else if err := ports.ValidateDurableHostProjection(observed); err != nil { | |||
| 364 | + | r.log.Debug("broker: observation is not persistable", "endpoint", result.endpoint, "err", err) | |||
| 365 | + | kind, availability, failed = domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |||
| 366 | + | } else { | |||
| 367 | + | r.hosts[result.endpoint] = observed | |||
| 368 | + | r.publishLocked(true) | |||
| 369 | + | return | |||
| 370 | + | } | |||
| 371 | + | } | |||
| 372 | + | // Every non-reachable outcome is a typed failure on the capped retry | |||
| 373 | + | // cadence: the last success, last-known inventory, and failure episode | |||
| 374 | + | // counter stay authoritative. | |||
| 375 | + | current.Availability = availability | |||
| 376 | + | if current.ConsecutiveFailures == 0 { | |||
| 377 | + | current.FailureEpisode++ | |||
| 378 | + | } | |||
| 379 | + | current.ConsecutiveFailures++ | |||
| 380 | + | current.LastFailure = domain.RemoteFailure{Kind: kind, Err: result.err} | |||
| 381 | + | delay := r.retryDelay(current.ConsecutiveFailures) | |||
| 382 | + | if r.demand > 0 { | |||
| 383 | + | delay = demandRetryDelay(r.demandFreshForRemote, current.ConsecutiveFailures) | |||
| 384 | + | } | |||
| 385 | + | current.NextDue = result.at.Add(r.jitter(delay, result.endpoint, result.attempt)) | |||
| 386 | + | r.log.Debug("broker_probe_failed", "endpoint", result.endpoint, "kind", kind, "failures", current.ConsecutiveFailures, "err", result.err, "cause", errors.Unwrap(result.err)) | |||
| 387 | + | r.hosts[result.endpoint] = current | |||
| 388 | + | r.publishLocked(true) | |||
| 389 | + | } | |||
| 390 | + | ||||
| 391 | + | // observationOutcome classifies a completed observation. It reports | |||
| 392 | + | // failed=false only for a confirmed reachable result; every other outcome | |||
| 393 | + | // carries the typed failure kind and the availability to publish. The caller | |||
| 394 | + | // additionally checks the adopted projection against the durable rules | |||
| 395 | + | // (ports.ValidateDurableHostProjection), which cover the session bound and | |||
| 396 | + | // catalogue validity for the exact incarnation and success time. | |||
| 397 | + | func observationOutcome(result probeResult) (domain.RemoteFailureKind, domain.RemoteAvailability, bool) { | |||
| 398 | + | if result.err != nil { | |||
| 399 | + | kind := domain.RemoteFailureTransport | |||
| 400 | + | var typed domain.RemoteFailure | |||
| 401 | + | var brokerErr ports.BrokerError | |||
| 402 | + | switch { | |||
| 403 | + | case errors.As(result.err, &typed) && typed.Kind != domain.RemoteFailureNone: | |||
| 404 | + | // The dialer classified the failure (for example SSH authentication). | |||
| 405 | + | kind = typed.Kind | |||
| 406 | + | case errors.As(result.err, &brokerErr) && (brokerErr.Code == ports.BrokerErrorConflictingPolicy || brokerErr.Code == ports.BrokerErrorIncompatible): | |||
| 407 | + | // SSH worked; the daemon's identity or version was refused. | |||
| 408 | + | kind = domain.RemoteFailureIncompatible | |||
| 409 | + | case errors.Is(result.err, context.DeadlineExceeded): | |||
| 410 | + | kind = domain.RemoteFailureTimeout | |||
| 411 | + | } | |||
| 412 | + | return kind, availabilityFor(kind), true | |||
| 413 | + | } | |||
| 414 | + | switch result.snapshot.Availability { | |||
| 415 | + | case domain.RemoteAvailabilityReachable: | |||
| 416 | + | return domain.RemoteFailureNone, domain.RemoteAvailabilityReachable, false | |||
| 417 | + | case domain.RemoteAvailabilityNoDaemon: | |||
| 418 | + | return domain.RemoteFailureNone, domain.RemoteAvailabilityNoDaemon, false | |||
| 419 | + | case domain.RemoteAvailabilityIncompatible: | |||
| 420 | + | return domain.RemoteFailureIncompatible, domain.RemoteAvailabilityIncompatible, true | |||
| 421 | + | case domain.RemoteAvailabilityAuthFailed: | |||
| 422 | + | return domain.RemoteFailureAuthentication, domain.RemoteAvailabilityAuthFailed, true | |||
| 423 | + | case domain.RemoteAvailabilityInvalidResponse: | |||
| 424 | + | return domain.RemoteFailureInvalidResponse, domain.RemoteAvailabilityInvalidResponse, true | |||
| 425 | + | default: | |||
| 426 | + | // Unknown or Unreachable without an error still means reachability was | |||
| 427 | + | // not confirmed, so it retries on the typed backoff cadence. | |||
| 428 | + | return domain.RemoteFailureTransport, domain.RemoteAvailabilityUnreachable, true | |||
| 429 | + | } | |||
| 430 | + | } | |||
| 431 | + | ||||
| 432 | + | // availabilityFor maps a typed failure kind to the published availability. | |||
| 433 | + | // The typed kind is preserved separately for notice policy. | |||
| 434 | + | func availabilityFor(kind domain.RemoteFailureKind) domain.RemoteAvailability { | |||
| 435 | + | switch kind { | |||
| 436 | + | case domain.RemoteFailureAuthentication: | |||
| 437 | + | return domain.RemoteAvailabilityAuthFailed | |||
| 438 | + | case domain.RemoteFailureIncompatible: | |||
| 439 | + | return domain.RemoteAvailabilityIncompatible | |||
| 440 | + | case domain.RemoteFailureInvalidResponse: | |||
| 441 | + | return domain.RemoteAvailabilityInvalidResponse | |||
| 442 | + | default: | |||
| 443 | + | return domain.RemoteAvailabilityUnreachable | |||
| 444 | + | } | |||
| 445 | + | } | |||
| 446 | + | ||||
| 447 | + | // demandRetryDelay is the watched-host failure backoff: the demand cadence | |||
| 448 | + | // doubled per consecutive failure, capped at demandRetryMax. | |||
| 449 | + | func demandRetryDelay(base time.Duration, failures uint) time.Duration { | |||
| 450 | + | delay := base | |||
| 451 | + | for i := uint(1); i < failures && delay < demandRetryMax; i++ { | |||
| 452 | + | delay *= 2 | |||
| 453 | + | } | |||
| 454 | + | return min(delay, max(base, demandRetryMax)) | |||
| 455 | + | } | |||
| 456 | + | ||||
| 457 | + | // retryDelay is the capped exponential retry cadence for a failure streak. | |||
| 458 | + | func (r *Registry) retryDelay(failures uint) time.Duration { | |||
| 459 | + | delay := r.retryBase | |||
| 460 | + | for i := uint(1); i < failures; i++ { | |||
| 461 | + | if delay >= r.retryMax { | |||
| 462 | + | return r.retryMax | |||
| 463 | + | } | |||
| 464 | + | delay *= 2 | |||
| 465 | + | } | |||
| 466 | + | if delay > r.retryMax { | |||
| 467 | + | return r.retryMax | |||
| 468 | + | } | |||
| 469 | + | return delay | |||
| 470 | + | } | |||
| 471 | + | ||||
| 472 | + | // jittered spreads a base interval deterministically across endpoint and | |||
| 473 | + | // attempt: identical inputs always yield the identical duration, and the | |||
| 474 | + | // result stays within 90%-110% of the base. No wall clock or global | |||
| 475 | + | // randomness participates, so retry and healthy-refresh schedules are | |||
| 476 | + | // reproducible instead of racing a shared PRNG. | |||
| 477 | + | func jittered(base time.Duration, endpoint string, attempt uint64) time.Duration { | |||
| 478 | + | if base <= 0 { | |||
| 479 | + | return base | |||
| 480 | + | } | |||
| 481 | + | h := fnv.New32a() | |||
| 482 | + | h.Write([]byte(endpoint)) | |||
| 483 | + | var counter [8]byte | |||
| 484 | + | binary.BigEndian.PutUint64(counter[:], attempt) | |||
| 485 | + | h.Write(counter[:]) | |||
| 486 | + | spread := int64(h.Sum32() % 21) | |||
| 487 | + | return base*90/100 + time.Duration(spread)*base/100 | |||
| 488 | + | } | |||
| 489 | + | ||||
| 490 | + | func (r *Registry) nextDelay(now time.Time) time.Duration { | |||
| 491 | + | r.mu.Lock() | |||
| 492 | + | defer r.mu.Unlock() | |||
| 493 | + | if r.demand == 0 { | |||
| 494 | + | if r.localPending || len(r.pending) > 0 { | |||
| 495 | + | return 0 | |||
| 496 | + | } | |||
| 497 | + | return time.Hour | |||
| 498 | + | } | |||
| 499 | + | var earliest time.Time | |||
| 500 | + | if r.local != nil && r.localAttempt == nil { | |||
| 501 | + | if r.localHost.NextDue.IsZero() { | |||
| 502 | + | return 0 | |||
| 503 | + | } | |||
| 504 | + | if earliest.IsZero() || r.localHost.NextDue.Before(earliest) { | |||
| 505 | + | earliest = r.localHost.NextDue | |||
| 506 | + | } | |||
| 507 | + | } | |||
| 508 | + | for endpoint, host := range r.hosts { | |||
| 509 | + | // An observation already in flight is completed by its result, never | |||
| 510 | + | // by the schedule; waiting on it here would spin the run loop. | |||
| 511 | + | if _, inflight := r.inflight[endpoint]; inflight { | |||
| 512 | + | continue | |||
| 513 | + | } | |||
| 514 | + | if r.pending[endpoint] || host.NextDue.IsZero() { | |||
| 515 | + | return 0 | |||
| 516 | + | } | |||
| 517 | + | if earliest.IsZero() || host.NextDue.Before(earliest) { | |||
| 518 | + | earliest = host.NextDue | |||
| 519 | + | } | |||
| 520 | + | } | |||
| 521 | + | if earliest.IsZero() { | |||
| 522 | + | return time.Hour | |||
| 523 | + | } | |||
| 524 | + | if delay := earliest.Sub(now); delay > 0 { | |||
| 525 | + | return delay | |||
| 526 | + | } | |||
| 527 | + | return 0 | |||
| 528 | + | } | |||
A internal/usecase/broker/registry_publish.go +114 -0
| @@ -0,0 +1,114 @@ | |||
| 1 | + | package broker | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "sync" | |
| 5 | + | ||
| 6 | + | "github.com/bnema/vev/internal/ports" | |
| 7 | + | ) | |
| 8 | + | ||
| 9 | + | // Snapshot returns a defensive immutable copy without I/O. | |
| 10 | + | func (r *Registry) Snapshot() ports.BrokerSnapshot { return r.current.Load().Clone() } | |
| 11 | + | ||
| 12 | + | // Subscribe returns a capacity-one notification stream. Slow consumers never | |
| 13 | + | // block publication and always re-read the newest snapshot. | |
| 14 | + | func (r *Registry) Subscribe() ports.BrokerSubscription { | |
| 15 | + | s := &subscription{changed: make(chan struct{}, 1)} | |
| 16 | + | s.close = func() { | |
| 17 | + | r.mu.Lock() | |
| 18 | + | delete(r.subs, s) | |
| 19 | + | r.mu.Unlock() | |
| 20 | + | } | |
| 21 | + | r.mu.Lock() | |
| 22 | + | r.subs[s] = struct{}{} | |
| 23 | + | r.mu.Unlock() | |
| 24 | + | s.offer() | |
| 25 | + | return s | |
| 26 | + | } | |
| 27 | + | ||
| 28 | + | // revisionExhaustedLocked reports whether this epoch has no revision left to | |
| 29 | + | // publish, latching the exhausted state on first detection. Callers must hold | |
| 30 | + | // r.mu. | |
| 31 | + | func (r *Registry) revisionExhaustedLocked() bool { | |
| 32 | + | if r.revisionExhausted { | |
| 33 | + | return true | |
| 34 | + | } | |
| 35 | + | if r.revision != ^ports.BrokerRevision(0) { | |
| 36 | + | return false | |
| 37 | + | } | |
| 38 | + | r.revisionExhausted = true | |
| 39 | + | r.log.Error("broker: revision series exhausted; refusing further publication", "epoch", r.epoch) | |
| 40 | + | return true | |
| 41 | + | } | |
| 42 | + | ||
| 43 | + | func (r *Registry) publishLocked(persist bool) { | |
| 44 | + | // Fail closed on an exhausted series: a wrapped revision would be zero, and | |
| 45 | + | // a wrapped series would publish revisions below the newest one, either of | |
| 46 | + | // which silently freezes durable persistence. The last valid snapshot stays | |
| 47 | + | // published, nothing is stored, and no subscriber is woken for a publication | |
| 48 | + | // that did not happen. | |
| 49 | + | if r.revisionExhaustedLocked() { | |
| 50 | + | return | |
| 51 | + | } | |
| 52 | + | // Remote daemons are published in registration order. The broker's own | |
| 53 | + | // machine daemon, when a producer is configured, is the | |
| 54 | + | // prepended entry at index zero; without a producer the registry produces no | |
| 55 | + | // local observation and never invents one. | |
| 56 | + | daemons := make([]ports.BrokerDaemonObservation, 0, len(r.hosts)+1) | |
| 57 | + | if r.local != nil { | |
| 58 | + | daemons = append(daemons, r.localHost.Clone()) | |
| 59 | + | } | |
| 60 | + | for _, endpoint := range r.order { | |
| 61 | + | host, ok := r.hosts[endpoint] | |
| 62 | + | if !ok { | |
| 63 | + | continue | |
| 64 | + | } | |
| 65 | + | daemons = append(daemons, host.Clone()) | |
| 66 | + | } | |
| 67 | + | snapshot := ports.BrokerSnapshot{Epoch: r.epoch, Revision: r.revision + 1, Daemons: daemons, Removed: r.tombstonesLocked()} | |
| 68 | + | if err := snapshot.Validate(); err != nil { | |
| 69 | + | // Fail closed: a publication the wire refuses aborts every client | |
| 70 | + | // connection on subscribe, so an invalid projection is never published. | |
| 71 | + | // The last valid snapshot stays current, nothing is persisted, and the | |
| 72 | + | // revision series does not advance past a publication that never | |
| 73 | + | // happened. | |
| 74 | + | r.log.Error("broker: refused invalid snapshot publication", "revision", r.revision+1, "err", err) | |
| 75 | + | return | |
| 76 | + | } | |
| 77 | + | r.revision = snapshot.Revision | |
| 78 | + | r.current.Store(&snapshot) | |
| 79 | + | if persist { | |
| 80 | + | // enqueue only stages an immutable publication: rendering the durable | |
| 81 | + | // copy and every store call happen on the writer goroutine, never | |
| 82 | + | // under the registry lock and never on the probe schedule. | |
| 83 | + | r.store.enqueue(snapshot) | |
| 84 | + | } | |
| 85 | + | for sub := range r.subs { | |
| 86 | + | sub.offer() | |
| 87 | + | } | |
| 88 | + | } | |
| 89 | + | ||
| 90 | + | // durable returns the persistable copy of a publication, rendered by the | |
| 91 | + | // writer goroutine with the shared rule (ports.BrokerSnapshot.DurableProjection). | |
| 92 | + | // The registry may publish the process-local daemon, which is never durable, | |
| 93 | + | // so it is dropped first; the store would refuse it otherwise. Restored | |
| 94 | + | // observations regain policy from membership (see stampHostAuthority). | |
| 95 | + | func durable(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |
| 96 | + | return snapshot.WithoutLocal().DurableProjection() | |
| 97 | + | } | |
| 98 | + | ||
| 99 | + | type subscription struct { | |
| 100 | + | changed chan struct{} | |
| 101 | + | once sync.Once | |
| 102 | + | close func() | |
| 103 | + | } | |
| 104 | + | ||
| 105 | + | func (s *subscription) Changed() <-chan struct{} { return s.changed } | |
| 106 | + | ||
| 107 | + | func (s *subscription) Close() { s.once.Do(s.close) } | |
| 108 | + | ||
| 109 | + | func (s *subscription) offer() { | |
| 110 | + | select { | |
| 111 | + | case s.changed <- struct{}{}: | |
| 112 | + | default: | |
| 113 | + | } | |
| 114 | + | } | |
| @@ -0,0 +1,114 @@ | |||||
| 1 | + | package broker | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "sync" | |||
| 5 | + | ||||
| 6 | + | "github.com/bnema/vev/internal/ports" | |||
| 7 | + | ) | |||
| 8 | + | ||||
| 9 | + | // Snapshot returns a defensive immutable copy without I/O. | |||
| 10 | + | func (r *Registry) Snapshot() ports.BrokerSnapshot { return r.current.Load().Clone() } | |||
| 11 | + | ||||
| 12 | + | // Subscribe returns a capacity-one notification stream. Slow consumers never | |||
| 13 | + | // block publication and always re-read the newest snapshot. | |||
| 14 | + | func (r *Registry) Subscribe() ports.BrokerSubscription { | |||
| 15 | + | s := &subscription{changed: make(chan struct{}, 1)} | |||
| 16 | + | s.close = func() { | |||
| 17 | + | r.mu.Lock() | |||
| 18 | + | delete(r.subs, s) | |||
| 19 | + | r.mu.Unlock() | |||
| 20 | + | } | |||
| 21 | + | r.mu.Lock() | |||
| 22 | + | r.subs[s] = struct{}{} | |||
| 23 | + | r.mu.Unlock() | |||
| 24 | + | s.offer() | |||
| 25 | + | return s | |||
| 26 | + | } | |||
| 27 | + | ||||
| 28 | + | // revisionExhaustedLocked reports whether this epoch has no revision left to | |||
| 29 | + | // publish, latching the exhausted state on first detection. Callers must hold | |||
| 30 | + | // r.mu. | |||
| 31 | + | func (r *Registry) revisionExhaustedLocked() bool { | |||
| 32 | + | if r.revisionExhausted { | |||
| 33 | + | return true | |||
| 34 | + | } | |||
| 35 | + | if r.revision != ^ports.BrokerRevision(0) { | |||
| 36 | + | return false | |||
| 37 | + | } | |||
| 38 | + | r.revisionExhausted = true | |||
| 39 | + | r.log.Error("broker: revision series exhausted; refusing further publication", "epoch", r.epoch) | |||
| 40 | + | return true | |||
| 41 | + | } | |||
| 42 | + | ||||
| 43 | + | func (r *Registry) publishLocked(persist bool) { | |||
| 44 | + | // Fail closed on an exhausted series: a wrapped revision would be zero, and | |||
| 45 | + | // a wrapped series would publish revisions below the newest one, either of | |||
| 46 | + | // which silently freezes durable persistence. The last valid snapshot stays | |||
| 47 | + | // published, nothing is stored, and no subscriber is woken for a publication | |||
| 48 | + | // that did not happen. | |||
| 49 | + | if r.revisionExhaustedLocked() { | |||
| 50 | + | return | |||
| 51 | + | } | |||
| 52 | + | // Remote daemons are published in registration order. The broker's own | |||
| 53 | + | // machine daemon, when a producer is configured, is the | |||
| 54 | + | // prepended entry at index zero; without a producer the registry produces no | |||
| 55 | + | // local observation and never invents one. | |||
| 56 | + | daemons := make([]ports.BrokerDaemonObservation, 0, len(r.hosts)+1) | |||
| 57 | + | if r.local != nil { | |||
| 58 | + | daemons = append(daemons, r.localHost.Clone()) | |||
| 59 | + | } | |||
| 60 | + | for _, endpoint := range r.order { | |||
| 61 | + | host, ok := r.hosts[endpoint] | |||
| 62 | + | if !ok { | |||
| 63 | + | continue | |||
| 64 | + | } | |||
| 65 | + | daemons = append(daemons, host.Clone()) | |||
| 66 | + | } | |||
| 67 | + | snapshot := ports.BrokerSnapshot{Epoch: r.epoch, Revision: r.revision + 1, Daemons: daemons, Removed: r.tombstonesLocked()} | |||
| 68 | + | if err := snapshot.Validate(); err != nil { | |||
| 69 | + | // Fail closed: a publication the wire refuses aborts every client | |||
| 70 | + | // connection on subscribe, so an invalid projection is never published. | |||
| 71 | + | // The last valid snapshot stays current, nothing is persisted, and the | |||
| 72 | + | // revision series does not advance past a publication that never | |||
| 73 | + | // happened. | |||
| 74 | + | r.log.Error("broker: refused invalid snapshot publication", "revision", r.revision+1, "err", err) | |||
| 75 | + | return | |||
| 76 | + | } | |||
| 77 | + | r.revision = snapshot.Revision | |||
| 78 | + | r.current.Store(&snapshot) | |||
| 79 | + | if persist { | |||
| 80 | + | // enqueue only stages an immutable publication: rendering the durable | |||
| 81 | + | // copy and every store call happen on the writer goroutine, never | |||
| 82 | + | // under the registry lock and never on the probe schedule. | |||
| 83 | + | r.store.enqueue(snapshot) | |||
| 84 | + | } | |||
| 85 | + | for sub := range r.subs { | |||
| 86 | + | sub.offer() | |||
| 87 | + | } | |||
| 88 | + | } | |||
| 89 | + | ||||
| 90 | + | // durable returns the persistable copy of a publication, rendered by the | |||
| 91 | + | // writer goroutine with the shared rule (ports.BrokerSnapshot.DurableProjection). | |||
| 92 | + | // The registry may publish the process-local daemon, which is never durable, | |||
| 93 | + | // so it is dropped first; the store would refuse it otherwise. Restored | |||
| 94 | + | // observations regain policy from membership (see stampHostAuthority). | |||
| 95 | + | func durable(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |||
| 96 | + | return snapshot.WithoutLocal().DurableProjection() | |||
| 97 | + | } | |||
| 98 | + | ||||
| 99 | + | type subscription struct { | |||
| 100 | + | changed chan struct{} | |||
| 101 | + | once sync.Once | |||
| 102 | + | close func() | |||
| 103 | + | } | |||
| 104 | + | ||||
| 105 | + | func (s *subscription) Changed() <-chan struct{} { return s.changed } | |||
| 106 | + | ||||
| 107 | + | func (s *subscription) Close() { s.once.Do(s.close) } | |||
| 108 | + | ||||
| 109 | + | func (s *subscription) offer() { | |||
| 110 | + | select { | |||
| 111 | + | case s.changed <- struct{}{}: | |||
| 112 | + | default: | |||
| 113 | + | } | |||
| 114 | + | } | |||
A internal/usecase/broker/registry_snapshot_writer.go +160 -0
| @@ -0,0 +1,160 @@ | |||
| 1 | + | package broker | |
| 2 | + | ||
| 3 | + | import ( | |
| 4 | + | "log/slog" | |
| 5 | + | "reflect" | |
| 6 | + | "sync" | |
| 7 | + | "time" | |
| 8 | + | ||
| 9 | + | "github.com/bnema/vev/internal/ports" | |
| 10 | + | ) | |
| 11 | + | ||
| 12 | + | // snapshotWriter serializes durable snapshot writes off the registry lock. | |
| 13 | + | // Enqueue never blocks on store I/O and always coalesces to the newest | |
| 14 | + | // revision; Store calls run on one goroutine, so the durable store never sees | |
| 15 | + | // an out-of-order write. A failed write is logged and retried by the next | |
| 16 | + | // publication, so a store fault never stalls publication. Close flushes the | |
| 17 | + | // newest pending snapshot, stops the writer, and waits, so no write outlives | |
| 18 | + | // shutdown. | |
| 19 | + | type snapshotWriter struct { | |
| 20 | + | store ports.BrokerSnapshotStore | |
| 21 | + | log *slog.Logger | |
| 22 | + | ||
| 23 | + | mu sync.Mutex | |
| 24 | + | pending *ports.BrokerSnapshot | |
| 25 | + | written ports.BrokerRevision | |
| 26 | + | // last is the durable content of the newest successful write with its | |
| 27 | + | // revision and observation timestamps cleared. Only the writer goroutine | |
| 28 | + | // touches it. | |
| 29 | + | last *ports.BrokerSnapshot | |
| 30 | + | started bool | |
| 31 | + | closed bool | |
| 32 | + | wake chan struct{} | |
| 33 | + | stop chan struct{} | |
| 34 | + | done chan struct{} | |
| 35 | + | } | |
| 36 | + | ||
| 37 | + | func newSnapshotWriter(store ports.BrokerSnapshotStore, log *slog.Logger) *snapshotWriter { | |
| 38 | + | return &snapshotWriter{ | |
| 39 | + | store: store, log: log, | |
| 40 | + | wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), | |
| 41 | + | } | |
| 42 | + | } | |
| 43 | + | ||
| 44 | + | // enqueue stages a snapshot for durable persistence. It coalesces to the | |
| 45 | + | // newest revision, starts the writer on first use, and never blocks on the | |
| 46 | + | // store or copies session payloads. Enqueue after Close is refused: nothing is | |
| 47 | + | // written after shutdown. | |
| 48 | + | func (w *snapshotWriter) enqueue(snapshot ports.BrokerSnapshot) { | |
| 49 | + | w.mu.Lock() | |
| 50 | + | if w.closed { | |
| 51 | + | w.mu.Unlock() | |
| 52 | + | w.log.Debug("broker: durable snapshot write refused after shutdown", "revision", snapshot.Revision) | |
| 53 | + | return | |
| 54 | + | } | |
| 55 | + | if w.pending != nil { | |
| 56 | + | if snapshot.Revision <= w.pending.Revision { | |
| 57 | + | w.mu.Unlock() | |
| 58 | + | return | |
| 59 | + | } | |
| 60 | + | } else if snapshot.Revision <= w.written { | |
| 61 | + | w.mu.Unlock() | |
| 62 | + | return | |
| 63 | + | } | |
| 64 | + | // The publication is immutable after publication, so staging it is a value | |
| 65 | + | // copy; the defensive durable copy is rendered on the writer goroutine. | |
| 66 | + | w.pending = &snapshot | |
| 67 | + | if !w.started { | |
| 68 | + | w.started = true | |
| 69 | + | go w.run() | |
| 70 | + | } | |
| 71 | + | w.mu.Unlock() | |
| 72 | + | select { | |
| 73 | + | case w.wake <- struct{}{}: | |
| 74 | + | default: | |
| 75 | + | } | |
| 76 | + | } | |
| 77 | + | ||
| 78 | + | func (w *snapshotWriter) run() { | |
| 79 | + | defer close(w.done) | |
| 80 | + | for { | |
| 81 | + | select { | |
| 82 | + | case <-w.stop: | |
| 83 | + | w.flush() | |
| 84 | + | return | |
| 85 | + | case <-w.wake: | |
| 86 | + | w.flush() | |
| 87 | + | } | |
| 88 | + | } | |
| 89 | + | } | |
| 90 | + | ||
| 91 | + | // flush writes the newest staged snapshot, skipping revisions the store | |
| 92 | + | // already holds. | |
| 93 | + | func (w *snapshotWriter) flush() { | |
| 94 | + | for { | |
| 95 | + | w.mu.Lock() | |
| 96 | + | snapshot := w.pending | |
| 97 | + | written := w.written | |
| 98 | + | w.pending = nil | |
| 99 | + | w.mu.Unlock() | |
| 100 | + | if snapshot == nil { | |
| 101 | + | return | |
| 102 | + | } | |
| 103 | + | if snapshot.Revision <= written { | |
| 104 | + | continue | |
| 105 | + | } | |
| 106 | + | next := durable(*snapshot) | |
| 107 | + | key := durableContent(next) | |
| 108 | + | if w.last != nil && reflect.DeepEqual(*w.last, key) { | |
| 109 | + | // Only observation timestamps moved: the stored copy already | |
| 110 | + | // carries the same durable state, so skip the rewrite and fsync. | |
| 111 | + | // A restart re-observes every host before trusting freshness. | |
| 112 | + | continue | |
| 113 | + | } | |
| 114 | + | if err := w.store.Store(next); err != nil { | |
| 115 | + | w.last = nil | |
| 116 | + | w.log.Error("broker: durable snapshot write failed", "revision", snapshot.Revision, "err", err) | |
| 117 | + | continue | |
| 118 | + | } | |
| 119 | + | w.last = &key | |
| 120 | + | w.mu.Lock() | |
| 121 | + | if snapshot.Revision > w.written { | |
| 122 | + | w.written = snapshot.Revision | |
| 123 | + | } | |
| 124 | + | w.mu.Unlock() | |
| 125 | + | } | |
| 126 | + | } | |
| 127 | + | ||
| 128 | + | // durableContent returns the durable projection without the fields that change | |
| 129 | + | // on every routine observation: the revision and the attempt, success, and | |
| 130 | + | // due timestamps. | |
| 131 | + | func durableContent(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |
| 132 | + | out := snapshot | |
| 133 | + | out.Revision = 0 | |
| 134 | + | out.Daemons = make([]ports.BrokerDaemonObservation, len(snapshot.Daemons)) | |
| 135 | + | for i, daemon := range snapshot.Daemons { | |
| 136 | + | daemon.LastAttempt = time.Time{} | |
| 137 | + | daemon.LastSuccess = time.Time{} | |
| 138 | + | daemon.NextDue = time.Time{} | |
| 139 | + | out.Daemons[i] = daemon | |
| 140 | + | } | |
| 141 | + | return out | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | // close flushes the newest staged snapshot, stops the writer, and waits for | |
| 145 | + | // the drain to finish. It never returns before the in-flight write completes. | |
| 146 | + | func (w *snapshotWriter) close() { | |
| 147 | + | w.mu.Lock() | |
| 148 | + | if w.closed { | |
| 149 | + | w.mu.Unlock() | |
| 150 | + | return | |
| 151 | + | } | |
| 152 | + | w.closed = true | |
| 153 | + | started := w.started | |
| 154 | + | w.mu.Unlock() | |
| 155 | + | if !started { | |
| 156 | + | return | |
| 157 | + | } | |
| 158 | + | close(w.stop) | |
| 159 | + | <-w.done | |
| 160 | + | } | |
| @@ -0,0 +1,160 @@ | |||||
| 1 | + | package broker | |||
| 2 | + | ||||
| 3 | + | import ( | |||
| 4 | + | "log/slog" | |||
| 5 | + | "reflect" | |||
| 6 | + | "sync" | |||
| 7 | + | "time" | |||
| 8 | + | ||||
| 9 | + | "github.com/bnema/vev/internal/ports" | |||
| 10 | + | ) | |||
| 11 | + | ||||
| 12 | + | // snapshotWriter serializes durable snapshot writes off the registry lock. | |||
| 13 | + | // Enqueue never blocks on store I/O and always coalesces to the newest | |||
| 14 | + | // revision; Store calls run on one goroutine, so the durable store never sees | |||
| 15 | + | // an out-of-order write. A failed write is logged and retried by the next | |||
| 16 | + | // publication, so a store fault never stalls publication. Close flushes the | |||
| 17 | + | // newest pending snapshot, stops the writer, and waits, so no write outlives | |||
| 18 | + | // shutdown. | |||
| 19 | + | type snapshotWriter struct { | |||
| 20 | + | store ports.BrokerSnapshotStore | |||
| 21 | + | log *slog.Logger | |||
| 22 | + | ||||
| 23 | + | mu sync.Mutex | |||
| 24 | + | pending *ports.BrokerSnapshot | |||
| 25 | + | written ports.BrokerRevision | |||
| 26 | + | // last is the durable content of the newest successful write with its | |||
| 27 | + | // revision and observation timestamps cleared. Only the writer goroutine | |||
| 28 | + | // touches it. | |||
| 29 | + | last *ports.BrokerSnapshot | |||
| 30 | + | started bool | |||
| 31 | + | closed bool | |||
| 32 | + | wake chan struct{} | |||
| 33 | + | stop chan struct{} | |||
| 34 | + | done chan struct{} | |||
| 35 | + | } | |||
| 36 | + | ||||
| 37 | + | func newSnapshotWriter(store ports.BrokerSnapshotStore, log *slog.Logger) *snapshotWriter { | |||
| 38 | + | return &snapshotWriter{ | |||
| 39 | + | store: store, log: log, | |||
| 40 | + | wake: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), | |||
| 41 | + | } | |||
| 42 | + | } | |||
| 43 | + | ||||
| 44 | + | // enqueue stages a snapshot for durable persistence. It coalesces to the | |||
| 45 | + | // newest revision, starts the writer on first use, and never blocks on the | |||
| 46 | + | // store or copies session payloads. Enqueue after Close is refused: nothing is | |||
| 47 | + | // written after shutdown. | |||
| 48 | + | func (w *snapshotWriter) enqueue(snapshot ports.BrokerSnapshot) { | |||
| 49 | + | w.mu.Lock() | |||
| 50 | + | if w.closed { | |||
| 51 | + | w.mu.Unlock() | |||
| 52 | + | w.log.Debug("broker: durable snapshot write refused after shutdown", "revision", snapshot.Revision) | |||
| 53 | + | return | |||
| 54 | + | } | |||
| 55 | + | if w.pending != nil { | |||
| 56 | + | if snapshot.Revision <= w.pending.Revision { | |||
| 57 | + | w.mu.Unlock() | |||
| 58 | + | return | |||
| 59 | + | } | |||
| 60 | + | } else if snapshot.Revision <= w.written { | |||
| 61 | + | w.mu.Unlock() | |||
| 62 | + | return | |||
| 63 | + | } | |||
| 64 | + | // The publication is immutable after publication, so staging it is a value | |||
| 65 | + | // copy; the defensive durable copy is rendered on the writer goroutine. | |||
| 66 | + | w.pending = &snapshot | |||
| 67 | + | if !w.started { | |||
| 68 | + | w.started = true | |||
| 69 | + | go w.run() | |||
| 70 | + | } | |||
| 71 | + | w.mu.Unlock() | |||
| 72 | + | select { | |||
| 73 | + | case w.wake <- struct{}{}: | |||
| 74 | + | default: | |||
| 75 | + | } | |||
| 76 | + | } | |||
| 77 | + | ||||
| 78 | + | func (w *snapshotWriter) run() { | |||
| 79 | + | defer close(w.done) | |||
| 80 | + | for { | |||
| 81 | + | select { | |||
| 82 | + | case <-w.stop: | |||
| 83 | + | w.flush() | |||
| 84 | + | return | |||
| 85 | + | case <-w.wake: | |||
| 86 | + | w.flush() | |||
| 87 | + | } | |||
| 88 | + | } | |||
| 89 | + | } | |||
| 90 | + | ||||
| 91 | + | // flush writes the newest staged snapshot, skipping revisions the store | |||
| 92 | + | // already holds. | |||
| 93 | + | func (w *snapshotWriter) flush() { | |||
| 94 | + | for { | |||
| 95 | + | w.mu.Lock() | |||
| 96 | + | snapshot := w.pending | |||
| 97 | + | written := w.written | |||
| 98 | + | w.pending = nil | |||
| 99 | + | w.mu.Unlock() | |||
| 100 | + | if snapshot == nil { | |||
| 101 | + | return | |||
| 102 | + | } | |||
| 103 | + | if snapshot.Revision <= written { | |||
| 104 | + | continue | |||
| 105 | + | } | |||
| 106 | + | next := durable(*snapshot) | |||
| 107 | + | key := durableContent(next) | |||
| 108 | + | if w.last != nil && reflect.DeepEqual(*w.last, key) { | |||
| 109 | + | // Only observation timestamps moved: the stored copy already | |||
| 110 | + | // carries the same durable state, so skip the rewrite and fsync. | |||
| 111 | + | // A restart re-observes every host before trusting freshness. | |||
| 112 | + | continue | |||
| 113 | + | } | |||
| 114 | + | if err := w.store.Store(next); err != nil { | |||
| 115 | + | w.last = nil | |||
| 116 | + | w.log.Error("broker: durable snapshot write failed", "revision", snapshot.Revision, "err", err) | |||
| 117 | + | continue | |||
| 118 | + | } | |||
| 119 | + | w.last = &key | |||
| 120 | + | w.mu.Lock() | |||
| 121 | + | if snapshot.Revision > w.written { | |||
| 122 | + | w.written = snapshot.Revision | |||
| 123 | + | } | |||
| 124 | + | w.mu.Unlock() | |||
| 125 | + | } | |||
| 126 | + | } | |||
| 127 | + | ||||
| 128 | + | // durableContent returns the durable projection without the fields that change | |||
| 129 | + | // on every routine observation: the revision and the attempt, success, and | |||
| 130 | + | // due timestamps. | |||
| 131 | + | func durableContent(snapshot ports.BrokerSnapshot) ports.BrokerSnapshot { | |||
| 132 | + | out := snapshot | |||
| 133 | + | out.Revision = 0 | |||
| 134 | + | out.Daemons = make([]ports.BrokerDaemonObservation, len(snapshot.Daemons)) | |||
| 135 | + | for i, daemon := range snapshot.Daemons { | |||
| 136 | + | daemon.LastAttempt = time.Time{} | |||
| 137 | + | daemon.LastSuccess = time.Time{} | |||
| 138 | + | daemon.NextDue = time.Time{} | |||
| 139 | + | out.Daemons[i] = daemon | |||
| 140 | + | } | |||
| 141 | + | return out | |||
| 142 | + | } | |||
| 143 | + | ||||
| 144 | + | // close flushes the newest staged snapshot, stops the writer, and waits for | |||
| 145 | + | // the drain to finish. It never returns before the in-flight write completes. | |||
| 146 | + | func (w *snapshotWriter) close() { | |||
| 147 | + | w.mu.Lock() | |||
| 148 | + | if w.closed { | |||
| 149 | + | w.mu.Unlock() | |||
| 150 | + | return | |||
| 151 | + | } | |||
| 152 | + | w.closed = true | |||
| 153 | + | started := w.started | |||
| 154 | + | w.mu.Unlock() | |||
| 155 | + | if !started { | |||
| 156 | + | return | |||
| 157 | + | } | |||
| 158 | + | close(w.stop) | |||
| 159 | + | <-w.done | |||
| 160 | + | } | |||