diff --git a/bin/hive b/bin/hive index 60ea4682..df50a93e 100755 --- a/bin/hive +++ b/bin/hive @@ -5,12 +5,18 @@ $LOAD_PATH.unshift(File.expand_path("../lib", __dir__)) require "hive" require "hive/cli" require "json" +require "hive/invoked_binary" if ARGV == [ "--version" ] || ARGV == [ "-v" ] puts Hive::VERSION exit 0 end +if ARGV == [ "--install-fingerprint" ] + puts JSON.generate(Hive::InvokedBinary.loaded_identity) + exit 0 +end + JSON_TRUE_OPTIONS = %w[--json --json=true --json=TRUE --json=t --json=T].freeze JSON_FALSE_OPTIONS = %w[--no-json --skip-json --json=false --json=FALSE --json=f --json=F].freeze JSON_BOOLEAN_OPTIONS = (JSON_TRUE_OPTIONS + JSON_FALSE_OPTIONS).freeze diff --git a/config.example.yml b/config.example.yml index 808c665a..8365d980 100644 --- a/config.example.yml +++ b/config.example.yml @@ -1,6 +1,13 @@ --- registered_projects: [] +# Global kill switch for fail-closed recovery of the fixed Codex-auth and +# Claude-launch dependency marker allowlist. Attempts remain capped and +# audited per task/reason even across daemon restarts. +daemon: + auto_retry: + enabled: true + # Optional hosted screenshot links for 7-artifacts visual demos. # Run `hive connect screenote` to authorize uploads. HIVE_SCREENOTE_BASE_URL # can override the default service URL for staging/self-hosted deployments. diff --git a/lib/hive.rb b/lib/hive.rb index b22bfe0e..fa9268d5 100644 --- a/lib/hive.rb +++ b/lib/hive.rb @@ -62,7 +62,7 @@ module Hive # `dispatch_requests/` directory. See # `Hive::Daemon::DispatchRequestQueue` and # `Hive::Bot::DispatchRequestWriter`. - "hive-dispatch-request" => 2, + "hive-dispatch-request" => 3, # Reverse-direction notice the daemon writes for the bot to relay a # non-zero, bot-originated dispatch back to the originating Telegram # chat. See `Hive::Daemon::DispatchResultQueue` (ADV-1). diff --git a/lib/hive/brainstorm_parser.rb b/lib/hive/brainstorm_parser.rb index 88492e11..02cc8f9b 100644 --- a/lib/hive/brainstorm_parser.rb +++ b/lib/hive/brainstorm_parser.rb @@ -106,6 +106,10 @@ module Hive parsed.select { |question| question.answer.nil? } end + def answered_questions(parsed) + parsed.select(&:answered?) + end + def question_for(parsed, question_n) parsed.find { |question| question.n == question_n } end diff --git a/lib/hive/claude_launcher.rb b/lib/hive/claude_launcher.rb index 15b092ae..0df66845 100644 --- a/lib/hive/claude_launcher.rb +++ b/lib/hive/claude_launcher.rb @@ -488,7 +488,7 @@ module Hive mcp_config_path: nil, strict_mcp_config: false) command = [ "bash", - File.expand_path("scripts/interactive_claude_wrapper.sh", __dir__), + interactive_wrapper_path, "--cwd", cwd ] Array(add_dirs).each { |dir| command.concat([ "--add-dir", dir ]) } @@ -506,6 +506,30 @@ module Hive command end + def interactive_wrapper_path + File.expand_path("scripts/interactive_claude_wrapper.sh", __dir__) + end + + # Deterministic canary through the production readiness parser. A parser + # edit that accepts the trust wall or rejects the known idle pane disables + # Claude auto-recovery before it can clear any marker. + def readiness_self_check + accepted = <<~PANE + Claude Code + + ❯ + ⏵⏵ bypass permissions · for agents + PANE + rejected = <<~PANE + Claude Code + + Quick safety check + Yes, I trust this folder + ❯ + PANE + claude_ready_prompt?(accepted) && !claude_ready_prompt?(rejected) + end + def mcp_cli_flags(path, strict) return [] if path.to_s.strip.empty? diff --git a/lib/hive/commands/doctor.rb b/lib/hive/commands/doctor.rb index 99afbc08..3b6184b8 100644 --- a/lib/hive/commands/doctor.rb +++ b/lib/hive/commands/doctor.rb @@ -45,11 +45,13 @@ module Hive # JSON encoder. Returns `nil` before `#call` has populated it. attr_reader :rows - def initialize(config:, project_root:, json: false, output: $stdout) + def initialize(config:, project_root:, json: false, output: $stdout, + agent_health_checker: nil) @config = config @project_root = project_root @json = json @output = output + @agent_health_checker = agent_health_checker || method(:probe_agent_health) @rows = nil end @@ -71,6 +73,20 @@ module Hive EXIT_CONFIG_ERROR end + # Lightweight in-process subset used by daemon recovery. It checks only + # configured required stage/reviewer agents and skills; advisory runtime + # rows (qmd migration hints, exported-key warnings) retain CLI semantics + # but do not make dependency recovery unhealthy. + def required_checks + @rows = check_required_agents + check_stages + check_reviewers + { + healthy: @rows.none? { |row| failing_status?(row[:status]) }, + rows: @rows + } + rescue Hive::ConfigError, KeyError, ArgumentError => e + { healthy: false, rows: [], error: "#{e.class}: #{e.message}" } + end + private def failing_status?(status) @@ -81,6 +97,58 @@ module Hive STAGES.map { |stage| check_stage(stage) } end + def check_required_agents + required_agent_names.map do |agent_name| + profile = Hive::AgentProfiles.lookup(agent_name.to_sym, cfg: @config) + status, message = @agent_health_checker.call(profile) + { + kind: "agent", + stage: "agent", + label: "agent/#{agent_name}", + agent: agent_name, + configured_skill: profile.bin, + skill: "#{profile.bin} #{profile.version_flag}", + status: status.to_s, + message: message.to_s + } + end + end + + def required_agent_names + paths = Hive::Config::ROLE_AGENT_PATHS.reject do |path| + (path == %w[review ci agent] && @config.dig("review", "ci", "command").to_s.empty?) || + (path == %w[review triage agent] && @config.dig("review", "triage", "enabled") == false) || + (path == %w[review browser_test agent] && + @config.dig("review", "browser_test", "enabled") != true) + end + names = paths.filter_map do |path| + (@config.dig(*path) || Hive::Config::DEFAULTS.dig(*path)).to_s.then do |name| + name unless name.empty? + end + end + Array(@config.dig("review", "reviewers")).each do |spec| + next unless %w[agent codex_review].include?((spec["kind"] || "agent").to_s) + + name = spec["agent"].to_s + names << name unless name.empty? + end + names.uniq + end + + def probe_agent_health(profile) + version = profile.check_version! + profile.preflight! + [ + "present", + "#{profile.name} #{version} is runnable#{profile.min_version ? " (minimum #{profile.min_version})" : ""}" + ] + rescue Hive::AgentError => e + status = e.message.include?("below minimum") ? "version_too_old" : "missing" + [ status, e.message ] + rescue StandardError => e + [ "missing", "#{profile.name} preflight failed: #{e.class}: #{e.message}" ] + end + def check_tmux return [] unless Hive::Config.claude_mode(@config) == :tmux diff --git a/lib/hive/commands/markers.rb b/lib/hive/commands/markers.rb index f5df7df1..764b2721 100644 --- a/lib/hive/commands/markers.rb +++ b/lib/hive/commands/markers.rb @@ -6,6 +6,7 @@ require "hive/markers" require "hive/lock" require "hive/git_ops" require "hive/stages" +require "hive/events" module Hive module Commands @@ -96,7 +97,6 @@ module Hive folder = resolve_target task = Hive::Task.new(folder) validate_project_path_match!(task) - # Read+validate+rewrite must run under the same `.markers-lock` # that `Hive::Markers.set` uses. Otherwise a concurrent # `hive run` writes a NEW marker between our validation and our @@ -105,6 +105,7 @@ module Hive # `with_commit_lock` to serialise hive/state branch writes # against any concurrent committer (auto-heal, run loop). Hive::Markers.with_markers_lock(task.state_file) do + original_body = File.binread(task.state_file) marker = Hive::Markers.current(task.state_file) actual = marker.name.to_s.upcase unless actual == normalized @@ -115,6 +116,12 @@ module Hive match_attr_or_raise!(task, marker) Hive::Markers.remove_marker(task.state_file, marker.raw) + begin + emit_manual_clear_event(task, marker) + rescue StandardError + Hive::Markers.write_atomic(task.state_file, original_body) + raise + end end Hive::Lock.with_commit_lock(task.hive_state_path) do @@ -124,6 +131,28 @@ module Hive emit_success(task, normalized) end + def emit_manual_clear_event(task, marker) + Hive::Events.emit_required( + task_folder: task.folder, + slug: task.slug, + stage: "#{task.stage_index}-#{task.stage_name}", + event_type: :marker_cleared, + message: "manually cleared #{marker.name.to_s.upcase}", + details: { + "task_id" => task.id, + "task_slug" => task.slug, + "stage" => "#{task.stage_index}-#{task.stage_name}", + "marker_id" => marker.attrs["marker_id"], + "marker_reason" => marker.attrs["reason"], + "marker_name" => marker.name.to_s, + "actor" => "manual", + "action" => "marker_cleared", + "rationale" => "operator requested marker clear", + "timestamp" => Time.now.utc.iso8601(6) + } + ) + end + # Cross-process race guard. The TUI observes an ERROR marker at # time T and dispatches `hive markers clear` at T+1s. If a # concurrent `hive run` writes a fresh ERROR marker in that diff --git a/lib/hive/config.rb b/lib/hive/config.rb index c686876b..2fadf982 100644 --- a/lib/hive/config.rb +++ b/lib/hive/config.rb @@ -313,6 +313,9 @@ module Hive "daemon" => { "enabled" => false, "autostart" => false, + "auto_retry" => { + "enabled" => true + }, "poll_interval_sec" => 30, "fast_poll_sec" => 1, "edit_debounce_sec" => 30, @@ -2225,6 +2228,19 @@ module Hive "(true / false); got #{autostart.inspect} (#{autostart.class})" end + auto_retry = daemon["auto_retry"] + unless auto_retry.nil? || auto_retry.is_a?(Hash) + raise ConfigError, + "daemon.auto_retry in #{describe_source(source_path)} must be a Hash; " \ + "got #{auto_retry.class}" + end + auto_retry_enabled = auto_retry && auto_retry["enabled"] + unless auto_retry_enabled.nil? || auto_retry_enabled == true || auto_retry_enabled == false + raise ConfigError, + "daemon.auto_retry.enabled in #{describe_source(source_path)} must be a boolean " \ + "(true / false); got #{auto_retry_enabled.inspect} (#{auto_retry_enabled.class})" + end + DAEMON_NUMERIC_BOUNDS.each do |key, min| value = daemon[key] next if value.nil? diff --git a/lib/hive/daemon/auto_retry.rb b/lib/hive/daemon/auto_retry.rb new file mode 100644 index 00000000..8c61f352 --- /dev/null +++ b/lib/hive/daemon/auto_retry.rb @@ -0,0 +1,388 @@ +require "time" +require "hive/daemon/auto_retry_history" +require "hive/daemon/dispatch_request_queue" +require "hive/events" +require "hive/markers" + +module Hive + module Daemon + # Coordinates fail-closed recovery for the fixed dependency-failure + # allowlist. The policy proves provenance/work safety, the probe proves + # current health, and the history owns the durable attempt budget. + class AutoRetry + PROBE_FALLBACK_SEC = 600 + NEGATIVE_AUDIT_THROTTLE_SEC = 600 + + Result = Struct.new( + :action, :rationale, :request_id, :attempt, :fingerprint, + keyword_init: true + ) + + def initialize(policy:, probe_factory:, state_home:, history: AutoRetryHistory.new, + request_queue: DispatchRequestQueue, markers: Hive::Markers, + events: Hive::Events, audit: nil, dry_run: false) + @policy = policy + @probe_factory = probe_factory + @state_home = state_home + @history = history + @request_queue = request_queue + @markers = markers + @events = events + @audit = audit || ->(**_attrs) { } + @dry_run = dry_run + @probe_cache = {} + @negative_audits = {} + end + + def tick(rows, now: Time.now) + @tick_probe_cache = {} + evict_negative_audits(now) + seen = {} + Array(rows).filter_map do |row| + key = marker_key(row) + next if key && seen[key] + + seen[key] = true if key + process(row, now: now) + rescue StandardError => e + audit(row, action: "coordinator_error", + rationale: "#{e.class}: #{e.message}", transition: true, now: now) + Result.new(action: "coordinator_error", rationale: "#{e.class}: #{e.message}") + end + end + + private + + def process(row, now:) + policy = @policy.evaluate(row) + unless policy.eligible + unless policy.action == "not_current_terminal_error" + audit(row, action: policy.action, rationale: policy.rationale, now: now) + end + return Result.new(action: policy.action, rationale: policy.rationale) + end + + reason = policy.reason + pending = @dry_run ? nil : pending_retry_request(row, policy.marker_id) + budget = @history.budget(task_folder: row.folder, reason: reason, now: now) + unless budget.eligible || pending + audit(row, action: budget.action, rationale: budget.rationale, now: now) + return Result.new(action: budget.action, rationale: budget.rationale) + end + + aggregate = probe(row, policy, reason, now) + unless aggregate.healthy + rationale = unhealthy_rationale(aggregate) + audit(row, action: "dependency_unhealthy", rationale: rationale, + aggregate: aggregate, now: now) + return Result.new( + action: "dependency_unhealthy", rationale: rationale, + fingerprint: aggregate.fingerprint + ) + end + + if pending + attempt = @history.reserved_attempt( + task_folder: row.folder, reason: reason, marker_id: policy.marker_id + ) + unless attempt + rationale = "guarded request has no matching durable reservation" + audit(row, action: "history_invalid", rationale: rationale, + aggregate: aggregate, request_id: pending.request_id, + transition: true, now: now) + return Result.new(action: "history_invalid", rationale: rationale) + end + return complete_retry( + row, policy, aggregate, attempt, pending.request_id, now, + resumed: true + ) + end + + decision = @history.assess( + task_folder: row.folder, reason: reason, + fingerprint: aggregate.fingerprint, now: now + ) + unless decision.eligible + audit(row, action: decision.action, rationale: decision.rationale, + aggregate: aggregate, now: now) + return Result.new( + action: decision.action, rationale: decision.rationale, + fingerprint: aggregate.fingerprint + ) + end + + return dry_run_result(row, decision.attempt, aggregate, now) if @dry_run + + retry_task(row, policy, decision, aggregate, now) + end + + def retry_task(row, policy, decision, aggregate, now) + probes = probe_summaries(aggregate) + reserved = @history.reserve!( + task_folder: row.folder, + task_id: row.id, + slug: row.slug, + stage: row.stage, + marker_id: policy.marker_id, + reason: policy.reason, + fingerprint: aggregate.fingerprint, + attempt: decision.attempt, + probes: probes, + now: now + ) + unless reserved + rationale = "durable retry reservation could not be appended" + audit(row, action: "reservation_failed", rationale: rationale, + aggregate: aggregate, attempt: decision.attempt, + transition: true, now: now) + return Result.new(action: "reservation_failed", rationale: rationale) + end + audit(row, action: "retry_reserved", rationale: "durable retry attempt reserved", + aggregate: aggregate, attempt: decision.attempt, + transition: true, task_event: false, now: now) + + request_id = enqueue(row, policy.marker_id, now) + unless request_id + rationale = "dispatch request could not be persisted" + audit(row, action: "queue_failed", rationale: rationale, + aggregate: aggregate, attempt: decision.attempt, + transition: true, now: now) + return Result.new( + action: "queue_failed", rationale: rationale, + attempt: decision.attempt, fingerprint: aggregate.fingerprint + ) + end + + complete_retry( + row, policy, aggregate, decision.attempt, request_id, now, + resumed: false + ) + end + + def complete_retry(row, policy, aggregate, attempt, request_id, now, resumed:) + probes = probe_summaries(aggregate) + cleared = @markers.clear_current( + row.state_file, + expected_name: :error, + match_attrs: { "marker_id" => policy.marker_id } + ) do + emit_automatic_clear(row, policy, aggregate, attempt, request_id, probes, now) + end + unless cleared + withdrawn = @request_queue.remove_if_unclaimed( + request_id, state_home: @state_home + ) + rationale = withdrawn ? "marker changed; unclaimed request withdrawn" : + "marker changed after request was claimed" + audit(row, action: "marker_race", rationale: rationale, + aggregate: aggregate, attempt: attempt, + request_id: request_id, transition: true, now: now) + return Result.new( + action: "marker_race", rationale: rationale, + request_id: request_id, attempt: attempt, + fingerprint: aggregate.fingerprint + ) + end + + audit(row, action: "retry_queued", rationale: "marker cleared and same-stage run queued", + aggregate: aggregate, attempt: attempt, + request_id: request_id, transition: true, task_event: false, now: now) + Result.new( + action: "retry_queued", + rationale: resumed ? "resumed reserved request; marker cleared and same-stage run queued" : + "marker cleared and same-stage run queued", + request_id: request_id, + attempt: attempt, + fingerprint: aggregate.fingerprint + ) + end + + def enqueue(row, marker_id, now) + @request_queue.write_request!( + project: row.project, + slug: row.slug, + argv: [ + "hive", "run", row.slug, "--project", row.project, + "--stage", row.stage + ], + # The queue's daemon-originated producer identity is `healer`. + # The dedicated subsystem remains distinguishable by trigger and + # task/daemon audit actor without widening the wire enum. + requestor: "healer", + trigger: "recoverable_dependency_failure", + expected_cleared_marker_id: marker_id, + state_home: @state_home, + now: now + ) + rescue StandardError + nil + end + + def emit_automatic_clear(row, policy, aggregate, attempt, request_id, probes, now) + record = @events.emit_required( + task_folder: row.folder, + slug: row.slug, + stage: row.stage, + event_type: :marker_cleared, + message: "cleared #{policy.reason} marker for automatic retry", + details: base_details(row, now).merge( + "marker_id" => policy.marker_id, + "marker_reason" => policy.reason, + "actor" => "daemon_auto_retry", + "probes" => probes, + "health_fingerprint" => aggregate.fingerprint, + "attempt" => attempt, + "request_id" => request_id, + "action" => "retry_queued", + "rationale" => "marker cleared and same-stage run queued" + ) + ) + raise IOError, "automatic marker-clear audit was not persisted" if record.nil? + + record + end + + def dry_run_result(row, attempt, aggregate, now) + rationale = "health and safety gates pass; dry-run leaves state unchanged" + audit(row, action: "would_retry", rationale: rationale, + aggregate: aggregate, attempt: attempt, + transition: true, task_event: false, now: now) + Result.new( + action: "would_retry", rationale: rationale, + attempt: attempt, fingerprint: aggregate.fingerprint + ) + end + + def audit(row, action:, rationale:, now:, aggregate: nil, attempt: nil, + request_id: nil, transition: false, task_event: true) + details = base_details(row, now).merge( + "marker_id" => row.marker_attrs.to_h["marker_id"].to_s, + "marker_reason" => row.marker_attrs.to_h["reason"].to_s, + "actor" => "daemon_auto_retry", + "probes" => aggregate ? probe_summaries(aggregate) : [], + "health_fingerprint" => aggregate&.fingerprint, + "attempt" => attempt, + "request_id" => request_id, + "action" => action.to_s, + "rationale" => rationale.to_s + ) + return if !transition && negative_audit_throttled?(details, now) + + if task_event && !@dry_run && !row.folder.to_s.empty? + @events.emit( + task_folder: row.folder, slug: row.slug, stage: row.stage, + event_type: :auto_retry_decision, + message: "#{action}: #{rationale}", + details: details + ) + end + @audit.call(**details.transform_keys(&:to_sym), transition: transition) + rescue ArgumentError, JSON::GeneratorError + nil + end + + def base_details(row, now) + { + "task_id" => row.id, + "task_slug" => row.slug.to_s, + "project" => row.project.to_s, + "stage" => row.stage.to_s, + "timestamp" => now.utc.iso8601(6) + } + end + + def probe_summaries(aggregate) + Array(aggregate.probes).map do |probe| + { + "name" => probe.name.to_s, + "healthy" => probe.healthy == true, + "exit_status" => probe.exit_status, + "timed_out" => probe.timed_out == true, + "duration_ms" => probe.duration_ms.to_i, + "rationale" => probe.rationale.to_s.byteslice(0, 256).to_s.scrub + } + end + end + + def unhealthy_rationale(aggregate) + failed = Array(aggregate.probes).reject(&:healthy).map(&:name) + failed.empty? ? "dependency probe aggregate is unhealthy" : + "unhealthy probes: #{failed.join(', ')}" + end + + def marker_key(row) + id = row.marker_attrs.to_h["marker_id"].to_s + return nil if id.empty? + + [ row.folder.to_s, row.marker_attrs.to_h["reason"].to_s, id ] + end + + def pending_retry_request(row, marker_id) + expected_argv = [ + "hive", "run", row.slug, "--project", row.project, + "--stage", row.stage + ] + @request_queue.pending(state_home: @state_home).find do |request| + request.project.to_s == row.project.to_s && + request.slug.to_s == row.slug.to_s && + request.argv == expected_argv && + request.requestor.to_s == "healer" && + request.trigger.to_s == "recoverable_dependency_failure" && + request.expected_cleared_marker_id.to_s == marker_id.to_s + end + rescue SystemCallError, IOError + nil + end + + def probe(row, policy, reason, now) + task_key = probe_task_key(row) + tick_key = [ row.project.to_s, reason.to_s ] + return @tick_probe_cache.fetch(tick_key) if @tick_probe_cache.key?(tick_key) + + runner = @probe_factory.call(row, policy) + cheap = runner.cheap_fingerprint(reason: reason) + context_key = [ task_key, reason.to_s ] + cached = @probe_cache[context_key] + aggregate = if cached && cached[:cheap] == cheap && + (now - cached[:checked_at]) < PROBE_FALLBACK_SEC + cached[:aggregate] + else + fresh = runner.call(reason: reason) + @probe_cache[context_key] = { + cheap: cheap, checked_at: now, aggregate: fresh + } + fresh + end + @tick_probe_cache[tick_key] = aggregate + end + + def probe_task_key(row) + [ + row.project.to_s, + row.id.to_s, + row.slug.to_s, + row.folder.to_s + ] + end + + def evict_negative_audits(now) + @negative_audits.delete_if do |_key, recorded_at| + (now - recorded_at) >= NEGATIVE_AUDIT_THROTTLE_SEC + end + end + + def negative_audit_throttled?(details, now) + key = [ + details["task_slug"], details["stage"], details["marker_id"], + details["marker_reason"], details["health_fingerprint"], + details["action"], details["rationale"] + ] + previous = @negative_audits[key] + return true if previous && (now - previous) < NEGATIVE_AUDIT_THROTTLE_SEC + + @negative_audits[key] = now + false + end + end + end +end diff --git a/lib/hive/daemon/auto_retry_history.rb b/lib/hive/daemon/auto_retry_history.rb new file mode 100644 index 00000000..ec855988 --- /dev/null +++ b/lib/hive/daemon/auto_retry_history.rb @@ -0,0 +1,183 @@ +require "json" +require "time" +require "hive/events" + +module Hive + module Daemon + # Durable, append-only retry budget reconstructed from task events. + # Only a successful manual marker clear starts a new reason episode. + class AutoRetryHistory + MAX_ATTEMPTS = 2 + SECOND_ATTEMPT_BACKOFF_SEC = 1800 + + Decision = Struct.new( + :eligible, :attempt, :action, :rationale, :previous_fingerprint, + :previous_reserved_at, + keyword_init: true + ) + + # Cheap pre-probe gate. It prevents external health commands from + # running while the second-attempt backoff is active or after the + # durable episode is exhausted. Fingerprint comparison remains in + # #assess after a fresh/cached aggregate is available. + def budget(task_folder:, reason:, now: Time.now) + reservations = episode_reservations(task_folder, reason) + return invalid_decision(reservations) if reservations.is_a?(String) + + case reservations.length + when 0 + Decision.new(eligible: true, attempt: 1, action: "retry_eligible", + rationale: "first automatic attempt is available") + when 1 + reserved_at = Time.iso8601(reservations.first.fetch("timestamp")) + if (now - reserved_at) < SECOND_ATTEMPT_BACKOFF_SEC + Decision.new( + eligible: false, action: "retry_backoff", + rationale: "second attempt requires a 30-minute backoff", + previous_fingerprint: reservations.first.fetch("health_fingerprint"), + previous_reserved_at: reserved_at + ) + else + Decision.new( + eligible: true, attempt: 2, action: "retry_probe_due", + rationale: "second-attempt backoff elapsed", + previous_fingerprint: reservations.first.fetch("health_fingerprint"), + previous_reserved_at: reserved_at + ) + end + else + Decision.new(eligible: false, attempt: nil, action: "retry_exhausted", + rationale: "two automatic attempts are already reserved") + end + rescue KeyError, ArgumentError + invalid_decision("malformed first reservation") + end + + def assess(task_folder:, reason:, fingerprint:, now: Time.now) + reservations = episode_reservations(task_folder, reason) + return invalid_decision(reservations) if reservations.is_a?(String) + + case reservations.length + when 0 + Decision.new(eligible: true, attempt: 1, action: "retry_eligible", + rationale: "first automatic attempt is available") + when 1 + assess_second(reservations.first, fingerprint, now) + else + Decision.new(eligible: false, attempt: nil, action: "retry_exhausted", + rationale: "two automatic attempts are already reserved") + end + end + + def reserved_attempt(task_folder:, reason:, marker_id:) + reservations = episode_reservations(task_folder, reason) + return nil if reservations.is_a?(String) + + reservation = reservations.reverse.find do |details| + details["marker_id"].to_s == marker_id.to_s + end + reservation && reservation["attempt"] + end + + def reserve!(task_folder:, task_id: nil, slug:, stage:, marker_id:, reason:, fingerprint:, + attempt:, probes:, now: Time.now) + record = Hive::Events.emit_required( + task_folder: task_folder, + slug: slug, + stage: stage, + event_type: :auto_retry_reserved, + message: "reserved automatic retry attempt #{attempt} for #{reason}", + details: { + "task_id" => task_id, + "task_slug" => slug, + "stage" => stage, + "marker_id" => marker_id, + "marker_reason" => reason, + "actor" => "daemon_auto_retry", + "probes" => probes, + "health_fingerprint" => fingerprint, + "attempt" => attempt, + "action" => "retry_reserved", + "rationale" => "health and safety gates passed", + "timestamp" => now.utc.iso8601(6) + } + ) + !record.nil? + rescue ArgumentError, JSON::GeneratorError, SystemCallError, IOError + false + end + + private + + def assess_second(first, fingerprint, now) + previous_fingerprint = first.fetch("health_fingerprint") + reserved_at = Time.iso8601(first.fetch("timestamp")) + if previous_fingerprint == fingerprint + return Decision.new( + eligible: false, action: "health_unchanged", + rationale: "second attempt requires a changed health fingerprint", + previous_fingerprint: previous_fingerprint, + previous_reserved_at: reserved_at + ) + end + if (now - reserved_at) < SECOND_ATTEMPT_BACKOFF_SEC + return Decision.new( + eligible: false, action: "retry_backoff", + rationale: "second attempt requires a 30-minute backoff", + previous_fingerprint: previous_fingerprint, + previous_reserved_at: reserved_at + ) + end + + Decision.new( + eligible: true, attempt: 2, action: "retry_eligible", + rationale: "health changed and the 30-minute backoff elapsed", + previous_fingerprint: previous_fingerprint, + previous_reserved_at: reserved_at + ) + rescue KeyError, ArgumentError + invalid_decision("malformed first reservation") + end + + def episode_reservations(task_folder, reason) + path = File.join(task_folder, "events.jsonl") + return [] unless File.exist?(path) + + events = File.readlines(path, chomp: true).reject(&:empty?).map { |line| JSON.parse(line) } + reset_index = events.rindex do |event| + manual_reset_for_reason?(event, reason) + end + episode = reset_index ? events[(reset_index + 1)..] : events + reservations = episode.select do |event| + event["event_type"] == "auto_retry_reserved" && + event.dig("details", "marker_reason").to_s == reason.to_s + end + return "malformed retry reservation" unless reservations.all? { |event| valid_reservation?(event) } + + reservations.map { |event| event.fetch("details") } + rescue JSON::ParserError, SystemCallError, IOError + "malformed or unreadable event history" + end + + def manual_reset_for_reason?(event, reason) + event["event_type"] == "marker_cleared" && + event.dig("details", "actor") == "manual" && + event.dig("details", "marker_reason").to_s == reason.to_s + end + + def valid_reservation?(event) + details = event["details"] + details.is_a?(Hash) && + details["attempt"].is_a?(Integer) && + details["attempt"].between?(1, MAX_ATTEMPTS) && + !details["health_fingerprint"].to_s.empty? && + !details["timestamp"].to_s.empty? + end + + def invalid_decision(reason) + Decision.new(eligible: false, attempt: nil, action: "history_invalid", + rationale: reason.to_s) + end + end + end +end diff --git a/lib/hive/daemon/auto_retry_policy.rb b/lib/hive/daemon/auto_retry_policy.rb new file mode 100644 index 00000000..360d7c42 --- /dev/null +++ b/lib/hive/daemon/auto_retry_policy.rb @@ -0,0 +1,277 @@ +require "hive/brainstorm_parser" +require "hive/config" +require "hive/daemon/bounded_command" +require "hive/diagnostic_helpers" +require "hive/markers" +require "hive/task" +require "hive/worktree" + +module Hive + module Daemon + # Eligibility and work-safety half of dependency recovery. It never + # probes dependencies, appends events, clears markers, or queues work. + class AutoRetryPolicy + REASONS = %w[implementer_failed claude_launch_failed].freeze + CLAUDE_LAUNCH_STAGES = %w[ + 2-brainstorm 3-plan 4-execute 5-open-pr 7-artifacts 8-finalize + ].freeze + WORKTREE_STAGES = %w[ + 4-execute 5-open-pr 6-review 7-artifacts 8-finalize + ].freeze + CODEX_AUTH_STATUS = /\b401\b/.freeze + CODEX_AUTH_DIAGNOSTIC = /missing bearer or basic authentication(?:\s+in\s+header)?/i.freeze + CODEX_EXIT_MESSAGE = /\Aexit_code=\d+\z/.freeze + CODEX_AGENT = "codex".freeze + CODEX_LOG_BASENAME = /\Aexecute-impl-[A-Za-z0-9._-]+\.log\z/.freeze + CLAUDE_MARKER_WRITER = "stages_base_claude_launcher".freeze + WORKTREE_COMMAND_TIMEOUT_SEC = 10 + WORKTREE_CAPTURE_BYTES = 4096 + + Decision = Struct.new( + :eligible, :action, :rationale, :reason, :marker_id, :task, + keyword_init: true + ) + + def initialize(task_builder: ->(folder) { Hive::Task.new(folder) }, + command_runner: nil, + config_loader: ->(root) { Hive::Config.load(root) }) + @task_builder = task_builder + @config_loader = config_loader + @command_runner = command_runner || lambda do |argv:, chdir:, timeout_sec:| + Hive::Daemon::BoundedCommand.capture( + env: {}, argv: argv, chdir: chdir, + timeout_sec: timeout_sec, max_bytes: WORKTREE_CAPTURE_BYTES + ) + end + end + + def evaluate(row) + marker_attrs = row.marker_attrs.to_h + reason = marker_attrs["reason"].to_s + marker_id = marker_attrs["marker_id"].to_s + return deny("not_current_terminal_error", "row is not a current terminal ERROR") unless terminal_error?(row) + return deny("marker_id_missing", "current ERROR marker has no marker_id") if marker_id.empty? + return deny("reason_not_allowlisted", "marker reason is not recoverable") unless REASONS.include?(reason) + + task = @task_builder.call(row.folder) + diagnostic = classify_reason(reason, row, task) + return diagnostic unless diagnostic.eligible + + safety = safe_to_retry(row, task) + return safety unless safety.eligible + + Decision.new( + eligible: true, + action: "eligible", + rationale: "recognized dependency failure and stage output is safe", + reason: reason, + marker_id: marker_id, + task: task + ) + rescue StandardError => e + deny("policy_error", "#{e.class}: #{e.message}") + end + + private + + def terminal_error?(row) + row.marker.to_s == "error" && row.action.to_s == "error" && + !row.folder.to_s.empty? && !row.state_file.to_s.empty? + end + + def classify_reason(reason, row, task) + case reason + when "implementer_failed" then codex_auth_failure(row, task) + when "claude_launch_failed" then claude_launch_failure(row) + else deny("reason_not_allowlisted", "marker reason is not recoverable") + end + end + + def codex_auth_failure(row, task) + return deny("codex_wrong_stage", "Codex auth recovery is execute-only") unless row.stage.to_s == "4-execute" # coding-scoped: Codex implementer auth recovery + + attrs = row.marker_attrs.to_h + unless configured_execute_agent(task) == CODEX_AGENT && + attrs["agent"].to_s == CODEX_AGENT + return deny( + "codex_agent_mismatch", + "current execute failure is not attributed to the configured Codex agent" + ) + end + unless attrs["status"].to_s == "error" && + CODEX_EXIT_MESSAGE.match?(attrs["message"].to_s) + return deny( + "codex_execute_marker_invalid", + "marker is not the production execute failure shape" + ) + end + + log_basename = attrs["log_file"].to_s + unless CODEX_LOG_BASENAME.match?(log_basename) && + File.basename(log_basename) == log_basename + return deny( + "codex_log_episode_ambiguous", + "current execute marker does not identify its exact implementer log" + ) + end + log_path = File.join(task.log_dir, log_basename) + return deny("codex_log_missing", "current execute implementer log is missing") unless File.file?(log_path) + + log_text = Hive::DiagnosticHelpers.tail_file(log_path) + unless codex_auth_signature?(log_text) + return deny("codex_auth_signature_missing", "current execute log does not contain both auth signatures") + end + if File.mtime(log_path) > File.mtime(row.state_file) + return deny("codex_log_episode_ambiguous", "execute log is newer than the current marker") + end + + allow("codex_auth_recognized", "current execute marker and exact log prove Codex authentication failure") + rescue SystemCallError, IOError + deny("codex_log_unreadable", "current execute diagnostic cannot be read") + end + + def codex_auth_signature?(text) + text.match?(CODEX_AUTH_STATUS) && text.match?(CODEX_AUTH_DIAGNOSTIC) + end + + def configured_execute_agent(task) + config = @config_loader.call(task.project_root) + (config.dig("execute", "agent") || "claude").to_s + end + + def claude_launch_failure(row) + attrs = row.marker_attrs.to_h + unless CLAUDE_LAUNCH_STAGES.include?(row.stage.to_s) + return deny("claude_stage_unsupported", "stage does not produce the v1 Claude launcher marker") + end + unless attrs["writer"].to_s == CLAUDE_MARKER_WRITER && + attrs["exception_class"].to_s == "Hive::AgentError" && + !attrs["message"].to_s.empty? + return deny("claude_marker_unattributed", "marker lacks the Claude launcher boundary attribution") + end + + allow("claude_launch_recognized", "marker was written by the Claude launcher failure boundary") + end + + def safe_to_retry(row, task) + current = Hive::Markers.current(row.state_file) + unless current.name == :error && current.attrs["marker_id"].to_s == row.marker_attrs.to_h["marker_id"].to_s + return deny("marker_changed", "on-disk marker no longer matches the status row") + end + if terminal_success_present?(row.state_file) + return deny("terminal_success_present", "current stage already has terminal success output") + end + + case row.stage.to_s + when "2-brainstorm" then brainstorm_safe(row.state_file) # coding-scoped: coding brainstorm Q&A safety + when "3-plan" then plan_safe(row.state_file) # coding-scoped: coding plan content safety + when *WORKTREE_STAGES then worktree_safe(task) + else deny("stage_safety_unknown", "stage has no fail-closed safety predicate") + end + rescue SystemCallError, IOError + deny("stage_state_unreadable", "stage output cannot be read safely") + end + + def brainstorm_safe(path) + questions = Hive::BrainstormParser.parse(path) + if Hive::BrainstormParser.answered_questions(questions).any? + return deny("brainstorm_answered", "brainstorm contains answered user Q&A") + end + + allow("brainstorm_untouched", "brainstorm has no answered user Q&A") + end + + def plan_safe(path) + text = File.read(path, encoding: "UTF-8") + body = strip_plan_scaffolding(text) + return deny("plan_substantive", "plan contains substantive content") unless body.empty? + + allow("plan_untouched", "plan contains only runner scaffolding") + end + + def strip_plan_scaffolding(text) + scrubbed = text.scrub + scrubbed.gsub(Hive::Markers::MARKER_RE, "") + .lines + .reject { |line| line.strip.empty? || line.match?(/\A\s*[#]{1,6}\s*(?:Plan|Feedback)?\s*\z/i) } + .join + .strip + end + + def worktree_safe(task) + pointer_path = task.worktree_yml_path + return deny("worktree_pointer_missing", "worktree.yml is missing") unless File.file?(pointer_path) + + pointer = Hive::Worktree.read_pointer(task.folder) + return deny("worktree_pointer_invalid", "worktree.yml cannot be parsed") unless pointer.is_a?(Hash) + + raw_path = pointer["path"].to_s + return deny("worktree_pointer_invalid", "worktree.yml has no path") if raw_path.empty? + + expected_root = Hive::Worktree.canonical_root(task.project_root) + path = Hive::Worktree.validate_pointer_path(raw_path, expected_root) + return deny("worktree_missing", "worktree directory is missing") unless File.directory?(path) + + listed = run_git( + task.project_root, + [ "git", "-C", task.project_root, "worktree", "list", "--porcelain" ] + ) + return listed if listed.is_a?(Decision) + + registered = listed.stdout.to_s.lines.filter_map do |line| + next unless line.start_with?("worktree ") + + Hive::Worktree.realpath_or_expand(line.delete_prefix("worktree ").strip) + end + unless registered.include?(Hive::Worktree.realpath_or_expand(path)) + return deny("worktree_unregistered", "worktree is not registered with the project repository") + end + + status = run_git( + path, + [ "git", "-C", path, "status", "--porcelain=v1", "--untracked-files=all" ] + ) + return status if status.is_a?(Decision) + unless status.stdout.to_s.empty? + return deny("worktree_dirty", "worktree contains uncommitted or untracked files") + end + + allow("worktree_clean", "worktree is completely clean") + rescue Hive::WorktreeError + deny("worktree_pointer_invalid", "worktree pointer is outside the configured root") + rescue SystemCallError, IOError + deny("worktree_status_failed", "worktree safety check could not run") + end + + def run_git(chdir, argv) + result = @command_runner.call( + argv: argv, chdir: chdir, timeout_sec: WORKTREE_COMMAND_TIMEOUT_SEC + ) + if result.timed_out + return deny("worktree_status_timeout", "git worktree safety check timed out") + end + unless result.status&.success? + message = result.stderr.to_s.lines.first.to_s.strip + return deny("worktree_status_failed", "git worktree safety check failed: #{message}") + end + + result + end + + def terminal_success_present?(path) + File.read(path, encoding: "UTF-8").to_enum(:scan, Hive::Markers::MARKER_RE).any? do + marker_name = Regexp.last_match[:name].downcase.to_sym + Hive::Markers::TERMINAL_MARKER_NAMES.include?(marker_name) + end + end + + def allow(action, rationale) + Decision.new(eligible: true, action: action, rationale: rationale) + end + + def deny(action, rationale) + Decision.new(eligible: false, action: action, rationale: rationale) + end + end + end +end diff --git a/lib/hive/daemon/auto_retry_probe.rb b/lib/hive/daemon/auto_retry_probe.rb new file mode 100644 index 00000000..6dba97fb --- /dev/null +++ b/lib/hive/daemon/auto_retry_probe.rb @@ -0,0 +1,407 @@ +require "digest" +require "json" +require "stringio" +require "tmpdir" +require "timeout" +require "time" +require "hive" +require "hive/agent_profiles" +require "hive/claude_launcher" +require "hive/commands/doctor" +require "hive/daemon/bounded_command" +require "hive/invoked_binary" +require "hive/secret_patterns" + +module Hive + module Daemon + # Bounded dependency probes used only after exact classification and work + # safety pass. Every result is redacted and output-capped before it leaves + # this class. + class AutoRetryProbe + MAX_CAPTURE_BYTES = 4096 + DOCTOR_TIMEOUT_SEC = 10 + SHORT_TIMEOUT_SEC = 10 + CODEX_SMOKE_TIMEOUT_SEC = 30 + + ProbeResult = Struct.new( + :name, :healthy, :exit_status, :timed_out, :duration_ms, + :stdout, :stderr, :rationale, + keyword_init: true + ) do + def to_h + { + "name" => name, + "healthy" => healthy, + "exit_status" => exit_status, + "timed_out" => timed_out, + "duration_ms" => duration_ms, + "stdout" => stdout, + "stderr" => stderr, + "rationale" => rationale + } + end + end + + Aggregate = Struct.new( + :reason, :healthy, :probes, :fingerprint, :cheap_fingerprint, + keyword_init: true + ) + + def initialize(config:, project_root:, hive_bin: nil, env: ENV, + command_runner: nil, doctor_factory: nil) + @config = config + @project_root = project_root + @env = env.to_h + @hive_bin = hive_bin || Hive::InvokedBinary.path(env: env) || "hive" + @command_runner = command_runner || method(:run_command) + @doctor_factory = doctor_factory || lambda { + Hive::Commands::Doctor.new( + config: @config, project_root: @project_root, output: StringIO.new + ) + } + end + + def call(reason:) + probes = [ doctor_probe ] + probes.concat(reason == "implementer_failed" ? codex_probes : claude_probes) + probes = probes.map { |probe| sanitize_probe(probe) } + cheap = safe_cheap_fingerprint(reason: reason) + fingerprint = health_fingerprint(reason: reason, probes: probes, cheap: cheap) + Aggregate.new( + reason: reason, + healthy: probes.all?(&:healthy), + probes: probes, + fingerprint: fingerprint, + cheap_fingerprint: cheap + ) + rescue StandardError => e + failure = local_result( + "probe_aggregate", false, "probe aggregate raised #{e.class}: #{e.message}" + ) + cheap = safe_cheap_fingerprint(reason: reason) + Aggregate.new( + reason: reason, healthy: false, probes: [ failure ], + fingerprint: health_fingerprint(reason: reason, probes: [ failure ], cheap: cheap), + cheap_fingerprint: cheap + ) + end + + def cheap_fingerprint(reason:) + signals = { + "reason" => reason.to_s, + "config_digest" => ::Digest::SHA256.hexdigest( + JSON.generate(relevant_config(reason)) + ), + "environment_selection" => environment_selection(reason), + "required_skills" => required_skill_signal, + } + signals.merge!(reason_specific_signals(reason)) + ::Digest::SHA256.hexdigest(JSON.generate(signals)) + end + + private + + def doctor_probe + outcome = required_checks_outcome + if outcome[:error] + rationale = outcome[:timed_out] ? "required doctor timed out" : + "required doctor raised #{outcome[:error].class}: #{outcome[:error].message}" + return local_result( + "required_doctor", false, rationale, + timed_out: outcome[:timed_out], started: outcome[:started] + ) + end + + result = outcome.fetch(:result) + healthy = result[:healthy] == true + local_result( + "required_doctor", healthy, + healthy ? "required agents and skills are present" : result[:error] || "required agent or skill is missing", + stdout: JSON.generate(Array(result[:rows])), + started: outcome[:started] + ) + end + + def codex_probes + profile = Hive::AgentProfiles.lookup(:codex, cfg: @config) + login = @command_runner.call( + name: "codex_login_status", + argv: [ profile_binary(profile), "login", "status" ], + env: @env, + chdir: @project_root, + timeout_sec: SHORT_TIMEOUT_SEC + ) + smoke = Dir.mktmpdir("hive-auto-retry-codex-") do |dir| + @command_runner.call( + name: "codex_exec_smoke", + argv: [ + profile_binary(profile), "exec", + "--sandbox", "read-only", + "--skip-git-repo-check", + "--ephemeral", + "--ignore-rules", + "-c", "shell_environment_policy.inherit=none", + "--json", "Respond with exactly OK. Do not read or write files." + ], + env: @env, + chdir: dir, + timeout_sec: CODEX_SMOKE_TIMEOUT_SEC + ) + end + [ login, smoke ] + rescue StandardError => e + [ local_result("codex_probe_setup", false, "Codex probe setup failed: #{e.class}: #{e.message}") ] + end + + def claude_probes + [ + wrapper_probe, + readiness_probe, + hive_identity_probe, + claude_binary_probe + ] + end + + def wrapper_probe + path = Hive::ClaudeLauncher.interactive_wrapper_path + healthy = File.file?(path) && File.executable?(path) + local_result( + "claude_wrapper", healthy, + healthy ? "packaged Claude wrapper is executable" : "packaged Claude wrapper is missing or not executable" + ) + rescue SystemCallError => e + local_result("claude_wrapper", false, "wrapper check failed: #{e.class}: #{e.message}") + end + + def readiness_probe + healthy = Hive::ClaudeLauncher.readiness_self_check + local_result( + "claude_readiness_detector", healthy, + healthy ? "readiness detector accepted and rejected its canaries" : "readiness detector self-check failed" + ) + end + + def hive_identity_probe + result = @command_runner.call( + name: "hive_install_identity", + argv: [ @hive_bin, "--install-fingerprint" ], + env: @env, + chdir: @project_root, + timeout_sec: SHORT_TIMEOUT_SEC + ) + return result unless result.healthy + + external = JSON.parse(result.stdout) + loaded = Hive::InvokedBinary.loaded_identity + matched = !loaded["fingerprint"].to_s.empty? && + external["fingerprint"].to_s == loaded["fingerprint"].to_s && + external["version"].to_s == Hive::VERSION + result.healthy = matched + result.rationale = matched ? "daemon and CLI installation identities match" : + "daemon and CLI installation identities differ" + result + rescue JSON::ParserError => e + local_result("hive_install_identity", false, "invalid CLI identity JSON: #{e.message}") + end + + def claude_binary_probe + profile = Hive::AgentProfiles.lookup(:claude, cfg: @config) + @command_runner.call( + name: "claude_binary", + argv: [ profile_binary(profile), profile.version_flag ], + env: @env, + chdir: @project_root, + timeout_sec: SHORT_TIMEOUT_SEC + ) + rescue StandardError => e + local_result("claude_binary", false, "Claude profile failed: #{e.class}: #{e.message}") + end + + def run_command(name:, argv:, env:, chdir:, timeout_sec:) + started = monotonic + command = Hive::Daemon::BoundedCommand.capture( + env: env, argv: argv, chdir: chdir, + timeout_sec: timeout_sec, max_bytes: MAX_CAPTURE_BYTES + ) + status = command.status + success = !command.timed_out && status&.success? + result( + name: name, healthy: success, exit_status: status&.exitstatus, + timed_out: command.timed_out, started: started, + stdout: command.stdout, stderr: command.stderr, + rationale: command.timed_out ? "command timed out after #{timeout_sec}s" : + (success ? "command succeeded" : "command exited #{status&.exitstatus}") + ) + rescue SystemCallError, IOError => e + result(name: name, healthy: false, started: started, stdout: "", stderr: "", + rationale: "command failed to start: #{e.class}: #{e.message}") + end + + def local_result(name, healthy, rationale, stdout: "", timed_out: false, started: monotonic) + result( + name: name, healthy: healthy, timed_out: timed_out, started: started, + stdout: stdout, stderr: "", rationale: rationale + ) + end + + def result(name:, healthy:, rationale:, started:, stdout: "", stderr: "", + exit_status: nil, timed_out: false) + ProbeResult.new( + name: name, + healthy: healthy, + exit_status: exit_status, + timed_out: timed_out, + duration_ms: ((monotonic - started) * 1000).round, + stdout: sanitize_output(stdout), + stderr: sanitize_output(stderr), + rationale: sanitize_output(rationale) + ) + end + + def sanitize_output(text) + redacted = Hive::SecretPatterns.redact(text.to_s) + redacted.byteslice(0, MAX_CAPTURE_BYTES).to_s.scrub + end + + def sanitize_probe(probe) + ProbeResult.new( + name: probe.name.to_s, + healthy: probe.healthy == true, + exit_status: probe.exit_status, + timed_out: probe.timed_out == true, + duration_ms: probe.duration_ms.to_i, + stdout: sanitize_output(probe.stdout), + stderr: sanitize_output(probe.stderr), + rationale: sanitize_output(probe.rationale) + ) + end + + def health_fingerprint(reason:, probes:, cheap:) + normalized = probes.map do |probe| + { + "name" => probe.name, + "healthy" => probe.healthy, + "exit_status" => probe.exit_status, + "timed_out" => probe.timed_out + } + end + ::Digest::SHA256.hexdigest(JSON.generate([ reason.to_s, cheap, normalized ])) + end + + def required_skill_signal + outcome = required_checks_outcome + return [ "error", outcome[:error].class.name, outcome[:timed_out] ] if outcome[:error] + + result = outcome.fetch(:result) + { + "healthy" => result[:healthy] == true, + "error" => result[:error].to_s, + "rows" => Array(result[:rows]) + } + end + + def required_checks_outcome + @required_checks_outcome ||= begin + started = monotonic + result = Timeout.timeout(DOCTOR_TIMEOUT_SEC) do + @doctor_factory.call.required_checks + end + { result: result, started: started, timed_out: false } + rescue Timeout::Error => e + { error: e, started: started, timed_out: true } + rescue StandardError => e + { error: e, started: started, timed_out: false } + end + end + + def relevant_config(reason) + common = { + "brainstorm" => @config["brainstorm"], + "plan" => @config["plan"], + "reviewers" => @config.dig("review", "reviewers") + } + if reason.to_s == "implementer_failed" + common.merge( + "agent" => @config.dig("agents", "codex"), + "execute" => @config["execute"] + ) + else + common.merge( + "agent" => @config.dig("agents", "claude"), + "claude" => @config["claude"] + ) + end + end + + def environment_selection(reason) + keys = if reason.to_s == "implementer_failed" + %w[HIVE_CODEX_BIN PATH] + else + %w[HIVE_CLAUDE_BIN HIVE_INVOKED_BIN PATH] + end + keys.to_h do |key| + [ key, ::Digest::SHA256.hexdigest(@env[key].to_s) ] + end + end + + def reason_specific_signals(reason) + if reason.to_s == "implementer_failed" + { + "codex_binary" => binary_signal(:codex) + } + else + { + "hive_identity" => Hive::InvokedBinary.loaded_identity["fingerprint"], + "claude_binary" => binary_signal(:claude), + "wrapper" => wrapper_signal + } + end + end + + def binary_signal(name) + profile = Hive::AgentProfiles.lookup(name, cfg: @config) + path = resolve_binary(profile_binary(profile)) + stat = File.stat(path) + [ path, stat.size, stat.mtime.to_i ] + rescue StandardError + [ name.to_s, "unresolved" ] + end + + def wrapper_signal + path = Hive::ClaudeLauncher.interactive_wrapper_path + stat = File.stat(path) + [ path, stat.size, stat.mtime.to_i, ::Digest::SHA256.file(path).hexdigest ] + rescue StandardError + [ "wrapper", "unresolved" ] + end + + def resolve_binary(binary) + return File.realpath(binary) if binary.include?(File::SEPARATOR) + + @env["PATH"].to_s.split(File::PATH_SEPARATOR).each do |dir| + candidate = File.join(dir, binary) + return File.realpath(candidate) if File.file?(candidate) && File.executable?(candidate) + end + binary + end + + def profile_binary(profile) + key = profile.env_bin_override_key + override = key && @env[key].to_s + override && !override.empty? ? override : profile.bin_default + end + + def safe_cheap_fingerprint(reason:) + cheap_fingerprint(reason: reason) + rescue StandardError => e + ::Digest::SHA256.hexdigest( + JSON.generate([ reason.to_s, "cheap_fingerprint_error", e.class.name ]) + ) + end + + def monotonic + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + end + end +end diff --git a/lib/hive/daemon/bounded_command.rb b/lib/hive/daemon/bounded_command.rb new file mode 100644 index 00000000..621c307a --- /dev/null +++ b/lib/hive/daemon/bounded_command.rb @@ -0,0 +1,141 @@ +module Hive + module Daemon + # Captures a subprocess through continuously-drained pipes. Only the first + # max_bytes per stream are retained; later bytes are discarded while the + # child runs, so a noisy or timed-out command cannot consume unbounded + # temporary storage. Readers are closed after the leader exits so a + # detached descendant that inherited stdout/stderr cannot hold capture open. + module BoundedCommand + Result = Struct.new(:stdout, :stderr, :status, :timed_out, keyword_init: true) + TERMINATE_GRACE_SEC = 1 + POLL_SEC = 0.01 + READER_JOIN_SEC = 0.1 + READ_CHUNK_BYTES = 16 * 1024 + + module_function + + def capture(env:, argv:, chdir:, timeout_sec:, max_bytes:) + stdout_reader, stdout_writer = IO.pipe + stderr_reader, stderr_writer = IO.pipe + stdout = +"" + stderr = +"" + readers = [ stdout_reader, stderr_reader ] + threads = [ + capture_stream(stdout_reader, stdout, max_bytes), + capture_stream(stderr_reader, stderr, max_bytes) + ] + + pid = Process.spawn( + env, *argv, + chdir: chdir, pgroup: true, in: File::NULL, + out: stdout_writer, err: stderr_writer + ) + stdout_writer.close + stderr_writer.close + status, timed_out = wait_with_deadline(pid, timeout_sec) + finish_capture(readers, threads) + Result.new( + stdout: stdout, + stderr: stderr, + status: status, + timed_out: timed_out + ) + ensure + [ stdout_writer, stderr_writer, stdout_reader, stderr_reader ].compact.each do |io| + io.close unless io.closed? + rescue IOError + nil + end + Array(threads).each do |thread| + next unless thread&.alive? + + thread.kill + thread.join + end + end + + def wait_with_deadline(pid, timeout_sec) + deadline = monotonic + timeout_sec.to_f + loop do + waited, status = Process.waitpid2(pid, Process::WNOHANG) + return [ status, false ] if waited + break if monotonic >= deadline + + sleep POLL_SEC + end + + [ terminate_group(pid), true ] + end + + def terminate_group(pid) + begin + Process.kill("TERM", -pid) + rescue Errno::ESRCH + return reap(pid) + end + deadline = monotonic + TERMINATE_GRACE_SEC + loop do + waited, status = Process.waitpid2(pid, Process::WNOHANG) + return status if waited + break if monotonic >= deadline + + sleep POLL_SEC + end + begin + Process.kill("KILL", -pid) + rescue Errno::ESRCH + nil + end + reap(pid) + rescue Errno::ECHILD + nil + end + + def reap(pid) + _waited, status = Process.waitpid2(pid) + status + rescue Errno::ECHILD + nil + end + + def capture_stream(io, buffer, max_bytes) + limit = [ max_bytes.to_i, 0 ].max + Thread.new do + loop do + chunk = io.readpartial(READ_CHUNK_BYTES) + remaining = limit - buffer.bytesize + buffer << chunk.byteslice(0, remaining) if remaining.positive? + end + rescue EOFError, IOError + nil + ensure + io.close unless io.closed? + end.tap { |thread| thread.report_on_exception = false } + end + + def finish_capture(readers, threads) + deadline = monotonic + READER_JOIN_SEC + threads.each do |thread| + remaining = deadline - monotonic + thread.join(remaining) if remaining.positive? + end + readers.each do |reader| + reader.close unless reader.closed? + rescue IOError + nil + end + threads.each do |thread| + thread.join(POLL_SEC) + next unless thread.alive? + + thread.kill + thread.join + end + end + + def monotonic + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + end + end +end diff --git a/lib/hive/daemon/dispatch_request_queue.rb b/lib/hive/daemon/dispatch_request_queue.rb index 00d7aacc..47d89c9f 100644 --- a/lib/hive/daemon/dispatch_request_queue.rb +++ b/lib/hive/daemon/dispatch_request_queue.rb @@ -13,7 +13,7 @@ module Hive module_function SCHEMA = "hive-dispatch-request".freeze - SCHEMA_VERSION = 2 + SCHEMA_VERSION = 3 ALLOWED_VERBS = %w[ run develop brainstorm plan review open-pr artifacts finalize @@ -27,10 +27,11 @@ module Hive SLUG_RE = /\A[a-z][a-z0-9-]{0,62}[a-z0-9]\z/ PROJECT_RE = /\A[A-Za-z0-9_.\-]+\z/ + MARKER_ID_RE = /\A[A-Za-z0-9._\-]+\z/ Request = Struct.new( :request_id, :created_at, :project, :slug, :argv, :requestor, - :chat_id, :update_id, :trigger, :path, + :chat_id, :update_id, :trigger, :expected_cleared_marker_id, :path, keyword_init: true ) @@ -50,12 +51,19 @@ module Hive def write_request!(project:, slug:, argv:, requestor: "bot", chat_id: nil, update_id: nil, trigger: nil, request_id: generate_request_id, + expected_cleared_marker_id: nil, state_home: Hive::Paths.state_home, now: Time.now) unless valid_argv?(argv) raise ArgumentError, "argv #{argv.inspect} is not allowlisted for dispatch requests" end raise ArgumentError, "project is required for dispatch requests" if project.to_s.empty? raise ArgumentError, "slug is required for dispatch requests" if slug.to_s.empty? + if !expected_cleared_marker_id.nil? && + (!expected_cleared_marker_id.is_a?(String) || + !MARKER_ID_RE.match?(expected_cleared_marker_id) || + expected_cleared_marker_id.to_s.bytesize > 128) + raise ArgumentError, "expected_cleared_marker_id is invalid" + end created_at = now.utc payload = { @@ -71,6 +79,9 @@ module Hive "update_id" => update_id, "trigger" => trigger.to_s } + unless expected_cleared_marker_id.nil? + payload["expected_cleared_marker_id"] = expected_cleared_marker_id.to_s + end dir = directory(state_home: state_home) filename = filename_for(created_at: created_at, request_id: request_id) @@ -360,6 +371,10 @@ module Hive end def expired?(request, now: Time.now, expiry_sec: EXPIRY_SEC) + if request.respond_to?(:expected_cleared_marker_id) && + !request.expected_cleared_marker_id.to_s.empty? + return false + end return false unless request.respond_to?(:created_at) created = request.created_at @@ -498,6 +513,14 @@ module Hive created_at = parse_time(data["created_at"]) return :invalid_created_at if created_at.nil? + expected_marker_id = data["expected_cleared_marker_id"] + if !expected_marker_id.nil? && + (!expected_marker_id.is_a?(String) || + !MARKER_ID_RE.match?(expected_marker_id) || + expected_marker_id.to_s.bytesize > 128) + return :invalid_expected_cleared_marker_id + end + Request.new( request_id: request_id, created_at: created_at, @@ -508,6 +531,7 @@ module Hive chat_id: data["chat_id"], update_id: data["update_id"], trigger: data["trigger"].to_s, + expected_cleared_marker_id: expected_marker_id, path: path ) end diff --git a/lib/hive/daemon/dispatcher.rb b/lib/hive/daemon/dispatcher.rb index 1f272c69..560b35d2 100644 --- a/lib/hive/daemon/dispatcher.rb +++ b/lib/hive/daemon/dispatcher.rb @@ -5,6 +5,7 @@ require "hive/config" require "hive/stages" require "hive/workflows" require "hive/task_action" +require "hive/task" require "hive/brainstorm_parser" require "hive/daemon/policy" require "hive/daemon/plan_approval" @@ -12,6 +13,9 @@ require "hive/daemon/concurrency_controller" require "hive/daemon/child_supervisor" require "hive/daemon/status_consumer" require "hive/daemon/stale_agent_healer" +require "hive/daemon/auto_retry" +require "hive/daemon/auto_retry_policy" +require "hive/daemon/auto_retry_probe" require "hive/daemon/display_name_backfiller" require "hive/daemon/task_id_backfiller" require "hive/daemon/dispatch_request_queue" @@ -22,6 +26,7 @@ require "hive/daemon/answer_digest_scheduler" require "hive/daemon/patrol_scheduler" require "hive/daemon/pr_merge_watcher" require "hive/lock" +require "hive/markers" require "hive/paths" require "hive/update_check" require "hive/update_check/state" @@ -60,7 +65,8 @@ module Hive merge_watcher: nil, patrol_scheduler: nil, digest_scheduler: nil, answer_digest_scheduler: nil, dry_run: false, update_state: nil, update_checker: nil, channel_detector: nil, - dispatch_request_state_home: nil, dispatch_result_state_home: nil) + dispatch_request_state_home: nil, dispatch_result_state_home: nil, + auto_retry: nil, auto_retry_factory: nil) @config = config @controller = controller @supervisor = supervisor @@ -167,6 +173,13 @@ module Hive # resolves the result home independently) never reads them (#251). @dispatch_request_state_home = dispatch_request_state_home @dispatch_result_state_home = dispatch_result_state_home + @auto_retry_factory = auto_retry_factory || -> { build_auto_retry } + @auto_retry_injected = !auto_retry.nil? && auto_retry_factory.nil? + @auto_retry = auto_retry || @auto_retry_factory.call + # Capture the loaded source identity at daemon construction, before a + # long-running process can observe an in-place install update. Ordinary + # short-lived CLI commands leave the expensive manifest hash lazy. + Hive::InvokedBinary.loaded_identity # `[project, slug] → last-logged error signature` for the # brainstorm-gate parse-error log dedup (see # `brainstorm_answers_pending?`). @@ -269,6 +282,12 @@ module Hive keeping_previous: true) end + # Dependency-health recovery owns only its fixed reason allowlist. + # It runs after stale marker normalization and before request + # consumption so a successful guarded clear can enter the normal + # queue path in this same tick. + run_auto_retry(result.rows, now: now) + # Self-heal tasks left showing their raw slug because name # generation never landed at `hive new`. Purely additive and # marker-free, so order relative to dispatch is irrelevant — but @@ -1337,6 +1356,15 @@ module Hive return end + if @legacy_layout_projects.key?(req.project) + @logger.event(:dispatch_request_blocked, + request_id: req.request_id, project: req.project, + slug: req.slug, reason: "legacy_layout_detected") + return + end + + return unless retry_marker_guard_allows?(req) + # Reverse the gate-evaluation order from the previous version: # check the cheap, deterministic running_task? gate first so an # in-flight slug doesn't incur the can_dispatch? scan. Per R-04 @@ -1380,9 +1408,10 @@ module Hive pid = dispatch_command( command, project: req.project, slug: req.slug, - # Request-driven runs don't carry a stage hint — the runner - # resolves the task's current stage at boot. Pass nil so the - # log line records `stage=nil` instead of inventing one. + # The runner resolves the task's current stage at boot. A guarded + # retry may carry `--stage` for exact state-file resolution, but + # request dispatch still records stage=nil rather than deriving + # controller metadata from arbitrary argv. stage: nil, state_file_mtime: state_file_path && File.exist?(state_file_path) ? File.mtime(state_file_path) : nil, state_file_path: state_file_path, @@ -1623,15 +1652,65 @@ module Hive # Best-effort lookup of the task's CURRENT state file so the # post-completion mtime refresh inside `reap_completed` has a - # path to stat. Mirrors `find_post_advance_state_file` but - # without the at-dispatch path (a request carries no stage hint). + # path to stat. Guarded auto-retry requests carry an exact `--stage`; + # ordinary requests retain the newest-state fallback. def resolve_request_state_file_path(req) project_entry = Hive::Config.find_project(req.project) return nil unless project_entry + expected = req.expected_cleared_marker_id.to_s + return resolve_guarded_retry_state_file(project_entry, req) unless expected.empty? + find_post_advance_state_file(project_entry["hive_state_path"], req.slug) end + def resolve_guarded_retry_state_file(project_entry, req) + stage_index = req.argv.each_index.find { |index| req.argv[index] == "--stage" } + return nil unless stage_index + + stage = req.argv[stage_index + 1].to_s + return nil if stage.empty? + + folder = File.join( + project_entry["hive_state_path"].to_s, + "stages", stage, req.slug.to_s + ) + return nil unless File.directory?(folder) + + Hive::Task.new(folder).state_file + rescue Hive::Error, ArgumentError + nil + end + + def retry_marker_guard_allows?(req) + expected = req.expected_cleared_marker_id.to_s + return true if expected.empty? + + state_file = resolve_request_state_file_path(req) + unless state_file && File.file?(state_file) + reject_request(req, reason: "marker_guard_state_unavailable") + return false + end + + marker = Hive::Markers.current(state_file) + return true if marker.name == :none + + if marker.name == :error && marker.attrs["marker_id"].to_s == expected + @logger.event( + :dispatch_request_blocked, + request_id: req.request_id, project: req.project, + slug: req.slug, reason: "marker_not_cleared" + ) + return false + end + + reject_request(req, reason: "marker_guard_mismatch") + false + rescue SystemCallError, IOError + reject_request(req, reason: "marker_guard_state_unavailable") + false + end + def dispatch_request_state_home @dispatch_request_state_home || Hive::Paths.state_home end @@ -1770,6 +1849,49 @@ module Hive @enabled_cache[project_name] = false end + def run_auto_retry(rows, now:) + return unless @daemon_cfg.dig("auto_retry", "enabled") != false + + eligible_rows = Array(rows).select do |row| + project_enabled?(row.project) && + !@legacy_layout_projects.key?(row.project) + end + @auto_retry.tick(eligible_rows, now: now) + rescue StandardError => e + @logger.event( + :auto_retry_error, + action: "coordinator_error", + rationale: "#{e.class}: #{e.message}", + timestamp: now.utc.iso8601(6) + ) + end + + def build_auto_retry + AutoRetry.new( + policy: AutoRetryPolicy.new, + probe_factory: lambda { |row, _decision| + entry = Hive::Config.find_project(row.project) + raise Hive::ConfigError, "project #{row.project.inspect} is not registered" unless entry + + root = entry["path"] + AutoRetryProbe.new( + config: Hive::Config.load(root), + project_root: root + ) + }, + state_home: dispatch_request_state_home, + dry_run: @dry_run, + audit: method(:audit_auto_retry) + ) + end + + def audit_auto_retry(transition:, **attrs) + @logger.event( + transition ? :auto_retry_action : :auto_retry_decision, + **attrs + ) + end + def reload_config! # PR-40 review P1 #2: rebase on the global ~/Dev/hive/config.yml's # daemon block, not bare DEFAULTS. @@ -1842,6 +1964,7 @@ module Hive logger: @logger, dry_run: @dry_run ) + @auto_retry = @auto_retry_factory.call unless @auto_retry_injected @enabled_cache.clear @logger.event(:config_reloaded) rescue Hive::ConfigError => e diff --git a/lib/hive/daemon/logger.rb b/lib/hive/daemon/logger.rb index 6af59dc9..afc20cf0 100644 --- a/lib/hive/daemon/logger.rb +++ b/lib/hive/daemon/logger.rb @@ -50,6 +50,9 @@ module Hive marker_heal_failed marker_heal_exhausted marker_heal_observer_missing + auto_retry_decision + auto_retry_action + auto_retry_error display_name_backfill update_available update_check_no_result diff --git a/lib/hive/daemon/status_consumer.rb b/lib/hive/daemon/status_consumer.rb index b483ad0a..48c28a2d 100644 --- a/lib/hive/daemon/status_consumer.rb +++ b/lib/hive/daemon/status_consumer.rb @@ -16,7 +16,7 @@ module Hive # the runner has written its claude_pid to the lock. The daemon # healer and dispatcher both need this so they don't race the runner # during the pre-claude window (issue #144). - Row = Struct.new(:project, :slug, :stage, :workflow, :marker, :marker_attrs, :folder, :state_file, + Row = Struct.new(:project, :id, :slug, :stage, :workflow, :marker, :marker_attrs, :folder, :state_file, :state_file_mtime, :action, :suggested_command, :claude_pid_alive, :live_task_lock, :diagnostic, :depends_on, :blocked_by, :dependency_stage, :blocked, @@ -180,6 +180,7 @@ module Hive Array(project_doc["tasks"]).each do |task| rows << Row.new( project: project, + id: task["id"], slug: task["slug"], stage: task["stage"], workflow: task["workflow"], diff --git a/lib/hive/events.rb b/lib/hive/events.rb index f8bca816..42cad8f9 100644 --- a/lib/hive/events.rb +++ b/lib/hive/events.rb @@ -15,6 +15,9 @@ module Hive round_complete clean_exit_auto_committed claude_completion_fallback + auto_retry_decision + auto_retry_reserved + marker_cleared ].freeze STATUS_TAIL_LINES = 20 @@ -32,6 +35,7 @@ module Hive # Keeps the full JSON line well below the single-write append budget # so concurrent emitters cannot interleave bytes within a record. MAX_MESSAGE_BYTES = 1024 + MAX_DETAILS_BYTES = 8192 MESSAGE_TRUNCATION_SUFFIX = "…[truncated]".freeze EM_DASH = "—".freeze @@ -46,7 +50,19 @@ module Hive # appenders via the inode lock; we cap message size (see # MAX_MESSAGE_BYTES) so the full line stays small and well-defined. # status.md is derived state and is rewritten with atomic rename. - def emit(task_folder:, slug:, stage:, event_type:, agent: nil, message: nil) + def emit(**kwargs) + emit_required(**kwargs) + rescue SystemCallError, IOError => e + warn "[hive.events] failed to emit #{kwargs[:event_type]} for #{kwargs[:task_folder]}: " \ + "#{e.class}: #{e.message}" + nil + end + + # Required variant for state transitions whose correctness depends on a + # durable event (retry reservations and manual retry-history resets). + # Append failures propagate to the caller. status.md is derived state, so + # a render failure is warned but does not invalidate an appended event. + def emit_required(task_folder:, slug:, stage:, event_type:, agent: nil, message: nil, details: nil) event_type = event_type.to_sym unless EVENT_TYPES.include?(event_type) raise ArgumentError, "unknown event_type #{event_type.inspect}; valid: #{EVENT_TYPES.inspect}" @@ -60,6 +76,7 @@ module Hive "event_type" => event_type.to_s, "message" => message.nil? ? nil : truncate_message(message.to_s) } + record["details"] = normalize_details(details) unless details.nil? FileUtils.mkdir_p(task_folder) events_path = File.join(task_folder, "events.jsonl") @@ -70,11 +87,30 @@ module Hive File.open(events_path, File::WRONLY | File::APPEND | File::CREAT, 0o644, encoding: "UTF-8") do |file| file.syswrite(line) end - render_status!(task_folder, record) + begin + render_status!(task_folder, record) + rescue SystemCallError, IOError => e + warn "[hive.events] failed to render status for #{task_folder}: #{e.class}: #{e.message}" + end record - rescue SystemCallError => e - warn "[hive.events] failed to emit #{event_type} for #{task_folder}: #{e.class}: #{e.message}" - nil + end + + def normalize_details(details) + unless details.is_a?(Hash) + raise ArgumentError, "event details must be a Hash; got #{details.class}" + end + + normalized = JSON.parse(JSON.generate(details)) + unless normalized.is_a?(Hash) + raise ArgumentError, "event details must encode as a JSON object" + end + if JSON.generate(normalized).bytesize > MAX_DETAILS_BYTES + raise ArgumentError, "event details exceed #{MAX_DETAILS_BYTES} bytes" + end + + normalized + rescue JSON::GeneratorError, TypeError => e + raise ArgumentError, "event details must contain only JSON values: #{e.message}" end def truncate_message(message) diff --git a/lib/hive/invoked_binary.rb b/lib/hive/invoked_binary.rb index 54eea962..ff3c9a4e 100644 --- a/lib/hive/invoked_binary.rb +++ b/lib/hive/invoked_binary.rb @@ -1,3 +1,6 @@ +require "digest" +require "json" + module Hive # Resolves the user-facing CLI path to bake into the systemd-user / # launchd daemon unit. Precedence: @@ -14,6 +17,7 @@ module Hive module InvokedBinary ENV_KEY = "HIVE_INVOKED_BIN".freeze VALID_NAMES = %w[hive hv].freeze + IDENTITY_MUTEX = Mutex.new module_function @@ -46,5 +50,52 @@ module Hive nil end + + # Capture the identity of the Hive source tree represented by this + # process. The manifest covers every Ruby file under lib/hive, not only + # hive.rb, so an in-place same-version update of a loaded component + # changes the identity seen by a newly started CLI. + def capture_loaded_identity(source: nil) + source ||= Hive.const_source_location(:VERSION)&.first + source = File.expand_path("../hive.rb", __dir__) if source.to_s.empty? + real_source = File.realpath(source) + root = File.expand_path("..", File.dirname(real_source)) + lib_root = File.dirname(real_source) + files = [ real_source ] + Dir.glob(File.join(lib_root, "hive", "**", "*.rb")) + digest = ::Digest::SHA256.new + files.map { |path| File.realpath(path) }.uniq.sort.each do |path| + digest.update(path.delete_prefix("#{root}#{File::SEPARATOR}")) + digest.update("\0") + digest.update(File.binread(path)) + digest.update("\0") + end + source_digest = digest.hexdigest + { + "version" => Hive::VERSION, + "install_root" => root, + "source_digest" => source_digest, + "fingerprint" => ::Digest::SHA256.hexdigest( + JSON.generate([ Hive::VERSION, root, source_digest ]) + ) + } + rescue SystemCallError + { + "version" => Hive::VERSION, + "install_root" => nil, + "source_digest" => nil, + "fingerprint" => nil + } + end + + def loaded_identity + identity = @loaded_identity + unless identity + IDENTITY_MUTEX.synchronize do + @loaded_identity ||= capture_loaded_identity.freeze + identity = @loaded_identity + end + end + identity.dup + end end end diff --git a/lib/hive/markers.rb b/lib/hive/markers.rb index a97b3c47..10692294 100644 --- a/lib/hive/markers.rb +++ b/lib/hive/markers.rb @@ -98,6 +98,9 @@ module Hive new_marker end + # When a block is supplied, run it after removal while retaining the + # marker lock. A raised side-effect restores the original state-file body + # before propagating, for transitions whose audit is required. def clear_current(state_file_path, expected_name:, match_attrs: {}) with_markers_lock(state_file_path) do marker = current(state_file_path) @@ -106,7 +109,16 @@ module Hive expected = match_attrs.to_h.transform_keys(&:to_s) return false unless expected.all? { |key, value| marker.attrs[key].to_s == value.to_s } + original_body = File.binread(state_file_path) remove_marker(state_file_path, marker.raw) + if block_given? + begin + yield marker + rescue StandardError + write_atomic(state_file_path, original_body) + raise + end + end true end end diff --git a/lib/hive/stages/base.rb b/lib/hive/stages/base.rb index 979b4800..cd1b2952 100644 --- a/lib/hive/stages/base.rb +++ b/lib/hive/stages/base.rb @@ -633,6 +633,7 @@ module Hive warn "[hive] claude launch failed: #{e.message}" Hive::Markers.set(task.state_file, :error, reason: "claude_launch_failed", + writer: "stages_base_claude_launcher", exception_class: e.class.name, message: e.message) { status: :error, error_message: e.message } diff --git a/lib/hive/stages/execute.rb b/lib/hive/stages/execute.rb index c3081b89..a05ddaa7 100644 --- a/lib/hive/stages/execute.rb +++ b/lib/hive/stages/execute.rb @@ -215,6 +215,8 @@ module Hive Hive::Markers.set(task.state_file, :error, reason: "implementer_failed", + agent: execute_agent_name(cfg), + log_file: implementation_log_basename(impl_result), status: impl_result&.fetch(:status, nil), message: impl_result&.fetch(:error_message, nil)) { commit: "implementer_failed", status: :error } @@ -233,6 +235,13 @@ module Hive nil end + def implementation_log_basename(impl_result) + path = impl_result&.fetch(:log_file, nil).to_s + return nil if path.empty? + + File.basename(path) + end + def spawn_implementation(task, cfg, worktree_path) plan_text = File.read(File.join(task.folder, "plan.md")) prompt = Hive::Stages::Base.render( diff --git a/schemas/hive-dispatch-request.v3.json b/schemas/hive-dispatch-request.v3.json new file mode 100644 index 00000000..288f8d44 --- /dev/null +++ b/schemas/hive-dispatch-request.v3.json @@ -0,0 +1,102 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ivankuznetsov/hive/blob/main/schemas/hive-dispatch-request.v3.json", + "title": "hive dispatch request (v3)", + "description": "One strict-version-matched JSON request under /dispatch_requests/. v3 adds expected_cleared_marker_id for daemon auto-retry requests. Guarded requests are durable beyond ordinary queue expiry and dispatch only after that exact ERROR marker is absent; a replacement marker rejects the request. The version bump makes older daemons reject, rather than unsafely ignore, this dispatch guard.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema", + "schema_version", + "request_id", + "created_at", + "project", + "slug", + "argv", + "requestor" + ], + "properties": { + "schema": { + "const": "hive-dispatch-request" + }, + "schema_version": { + "const": 3 + }, + "request_id": { + "type": "string", + "pattern": "^[a-f0-9]{8,32}$" + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "project": { + "type": "string", + "pattern": "^[A-Za-z0-9_.-]+$", + "minLength": 1 + }, + "slug": { + "type": "string", + "pattern": "^[a-z][a-z0-9-]{0,62}[a-z0-9]$" + }, + "argv": { + "type": "array", + "minItems": 2, + "items": { + "type": "string", + "minLength": 1 + } + }, + "requestor": { + "type": "string", + "enum": [ + "bot", + "healer" + ] + }, + "chat_id": { + "type": [ + "integer", + "null" + ] + }, + "update_id": { + "type": [ + "integer", + "null" + ] + }, + "trigger": { + "type": [ + "string", + "null" + ] + }, + "expected_cleared_marker_id": { + "type": "string", + "pattern": "^[A-Za-z0-9._-]+$", + "minLength": 1, + "maxLength": 128, + "description": "Optional auto-retry guard. The daemon dispatches only when this exact terminal ERROR marker has been cleared; a matching live marker blocks, and a replacement terminal marker rejects the request. Presence also disables ordinary 600-second pending expiry." + } + }, + "$defs": { + "ALLOWED_VERBS": { + "type": "array", + "items": { + "enum": [ + "run", + "develop", + "brainstorm", + "plan", + "review", + "open-pr", + "artifacts", + "finalize", + "archive", + "markers" + ] + } + } + } +} diff --git a/test/integration/cli_version_test.rb b/test/integration/cli_version_test.rb index e390718e..76d60bec 100644 --- a/test/integration/cli_version_test.rb +++ b/test/integration/cli_version_test.rb @@ -14,6 +14,14 @@ class CliVersionTest < Minitest::Test assert_equal "#{Hive::VERSION}\n", out end + def test_install_fingerprint_matches_loaded_identity + payload = JSON.parse( + run!(RbConfig.ruby, "-Ilib", "bin/hive", "--install-fingerprint") + ) + + assert_equal Hive::InvokedBinary.loaded_identity, payload + end + def test_bin_hive_help_after_option_value_shows_command_usage out, err, status = Open3.capture3( RbConfig.ruby, "-Ilib", "bin/hive", "approve", "--from", "2-brainstorm", "--help" diff --git a/test/integration/daemon_auto_retry_test.rb b/test/integration/daemon_auto_retry_test.rb new file mode 100644 index 00000000..5b6ef9fa --- /dev/null +++ b/test/integration/daemon_auto_retry_test.rb @@ -0,0 +1,391 @@ +require "test_helper" +require "shellwords" +require "yaml" +require "hive/config" +require "hive/daemon/auto_retry" +require "hive/daemon/auto_retry_history" +require "hive/daemon/auto_retry_policy" +require "hive/daemon/auto_retry_probe" +require "hive/daemon/concurrency_controller" +require "hive/daemon/dispatcher" +require "hive/daemon/dispatch_request_queue" +require "hive/daemon/status_consumer" +require "hive/invoked_binary" +require "hive/task_meta" + +class DaemonAutoRetryIntegrationTest < Minitest::Test + include HiveTestHelper + + HealthyDoctor = Struct.new(:unused) do + def required_checks + { healthy: true, rows: [] } + end + end + + class RecordingSupervisor + attr_reader :spawned + + def initialize + @spawned = [] + @next_pid = 100 + end + + def spawn(**attrs) + pid = @next_pid + @next_pid += 1 + @spawned << attrs.merge(pid: pid) + pid + end + + def reap_all(now: Time.now) + [] + end + + def reap_dry_run(now: Time.now) + [] + end + + def enforce_timeouts(now:) + [] + end + + def terminate_all(grace_sec: 600); end + + def update_timeouts(default_timeout_sec:, verb_timeouts:, kill_grace_sec:); end + + def in_flight_count + @spawned.length + end + end + + class RecordingLogger + attr_reader :events + + def initialize + @events = [] + end + + def event(name, **attrs) + @events << [ name, attrs ] + end + end + + def test_dispatcher_keeps_crash_seam_request_parked_until_codex_recovers + with_tmp_dir do |root| + fixture = build_execute_fixture(root) + login_flag = File.join(root, "codex-logged-in") + codex = write_executable(File.join(root, "bin", "codex"), <<~SH) + #!/bin/sh + if [ "$1" = "login" ]; then + test -f #{Shellwords.escape(login_flag)} + exit $? + fi + if [ "$1" = "exec" ]; then + printf '{"type":"result","result":"OK"}\\n' + exit 0 + fi + exit 2 + SH + hive, status_calls = write_status_hive(root, fixture) + seed_crash_seam_request(fixture) + dispatcher, supervisor, logger = build_dispatcher( + fixture, + hive_bin: hive, + env: ENV.to_h.merge("HIVE_CODEX_BIN" => codex) + ) + + with_registered_project(fixture) do + dispatcher.tick(now: fixture[:now]) + + assert_equal 1, File.readlines(status_calls).length + assert_empty supervisor.spawned + assert_equal :error, Hive::Markers.current(fixture[:state_file]).name + assert_equal 1, pending_requests(fixture).length + blocked = logger.events.find { |name, _attrs| name == :dispatch_request_blocked } + assert_equal "marker_not_cleared", blocked.last[:reason] + + File.write(login_flag, "healthy\n") + dispatcher.tick(now: fixture[:now] + 600) + end + + assert_equal 2, File.readlines(status_calls).length + assert_equal :none, Hive::Markers.current(fixture[:state_file]).name + assert_empty pending_requests(fixture) + assert_equal 1, supervisor.spawned.length + assert_equal( + "hive run retry-task --project demo --stage 4-execute", + supervisor.spawned.first.fetch(:command_string) + ) + cleared = read_events(fixture[:folder]).find do |event| + event["event_type"] == "marker_cleared" + end + assert_equal "crash-marker", cleared.dig("details", "marker_id") + end + end + + def test_dispatcher_runs_full_claude_probe_chain_and_dispatches_guarded_request + with_tmp_dir do |root| + fixture = build_brainstorm_fixture(root) + claude = write_executable(File.join(root, "bin", "claude"), <<~SH) + #!/bin/sh + echo 'Claude 2.1.999' + SH + hive, status_calls = write_status_hive(root, fixture, include_identity: true) + dispatcher, supervisor, _logger = build_dispatcher( + fixture, + hive_bin: hive, + env: ENV.to_h.merge("HIVE_CLAUDE_BIN" => claude) + ) + + with_registered_project(fixture) do + dispatcher.tick(now: fixture[:now]) + end + + assert_equal 1, File.readlines(status_calls).length + assert_equal :none, Hive::Markers.current(fixture[:state_file]).name + assert_empty pending_requests(fixture) + assert_equal 1, supervisor.spawned.length + reservation = read_events(fixture[:folder]).find do |event| + event["event_type"] == "auto_retry_reserved" + end + assert_equal 17, reservation.dig("details", "task_id") + assert_equal 1, reservation.dig("details", "attempt") + assert_equal 5, reservation.dig("details", "probes").length + end + end + + private + + def build_dispatcher(fixture, hive_bin:, env:) + config = Hive::Config.load(fixture[:project]) + auto_retry = Hive::Daemon::AutoRetry.new( + policy: Hive::Daemon::AutoRetryPolicy.new, + probe_factory: lambda { |_row, _decision| + Hive::Daemon::AutoRetryProbe.new( + config: config, + project_root: fixture[:project], + hive_bin: hive_bin, + env: env, + doctor_factory: -> { HealthyDoctor.new } + ) + }, + state_home: fixture[:state_home] + ) + controller = Hive::Daemon::ConcurrencyController.new( + max_concurrent_runs: 5, + max_concurrent_per_project: 5, + max_runs_per_day_per_project: 100 + ) + supervisor = RecordingSupervisor.new + logger = RecordingLogger.new + dispatcher = Hive::Daemon::Dispatcher.new( + config: config, + controller: controller, + supervisor: supervisor, + status_consumer: Hive::Daemon::StatusConsumer.new(hive_bin: hive_bin), + logger: logger, + dispatch_request_state_home: fixture[:state_home], + dispatch_result_state_home: fixture[:state_home], + auto_retry: auto_retry + ) + [ dispatcher, supervisor, logger ] + end + + def build_execute_fixture(root) + fixture = base_fixture(root, stage: "4-execute", state_name: "task.md") + worktree = File.join(root, "worktrees", "retry-task") + FileUtils.mkdir_p(File.dirname(worktree)) + run!("git", "-C", fixture[:project], "worktree", "add", "--quiet", + "-b", "retry-task", worktree) + File.write( + File.join(fixture[:folder], "worktree.yml"), + { "path" => worktree }.to_yaml + ) + + auth = "request failed with 401: Missing bearer or basic authentication in header" + log_dir = File.join(fixture[:project], ".hive-state", "logs", fixture[:slug]) + FileUtils.mkdir_p(log_dir) + log_name = "execute-impl-current.log" + log_file = File.join(log_dir, log_name) + File.write(log_file, auth) + attrs = { + "reason" => "implementer_failed", + "marker_id" => "crash-marker", + "agent" => "codex", + "log_file" => log_name, + "status" => "error", + "message" => "exit_code=1" + } + Hive::Markers.set(fixture[:state_file], :error, attrs) + fixture[:attrs] = attrs + fixture[:log_file] = log_file + fixture + end + + def build_brainstorm_fixture(root) + fixture = base_fixture(root, stage: "2-brainstorm", state_name: "brainstorm.md") + attrs = { + "reason" => "claude_launch_failed", + "marker_id" => "claude-marker", + "writer" => Hive::Daemon::AutoRetryPolicy::CLAUDE_MARKER_WRITER, + "exception_class" => "Hive::AgentError", + "message" => "Claude launcher failed before readiness" + } + Hive::Markers.set(fixture[:state_file], :error, attrs) + fixture[:attrs] = attrs + fixture + end + + def base_fixture(root, stage:, state_name:) + project = File.join(root, "demo") + FileUtils.mkdir_p(project) + run!("git", "-C", project, "init", "-b", "main", "--quiet") + run!("git", "-C", project, "config", "user.email", "test@example.com") + run!("git", "-C", project, "config", "user.name", "Test") + File.write(File.join(project, "README.md"), "fixture\n") + run!("git", "-C", project, "add", "README.md") + run!("git", "-C", project, "commit", "-m", "fixture", "--quiet") + folder = File.join(project, ".hive-state", "stages", stage, "retry-task") + FileUtils.mkdir_p(folder) + File.write( + File.join(project, ".hive-state", "config.yml"), + { + "daemon" => { + "enabled" => true, + "auto_retry" => { "enabled" => true } + }, + "execute" => { "agent" => "codex" }, + "worktree_root" => File.join(root, "worktrees") + }.to_yaml + ) + Hive::TaskMeta.write( + folder, + id: 17, + slug: "retry-task", + display_name: "Retry task" + ) + { + project: project, + hive_state: File.join(project, ".hive-state"), + folder: folder, + state_file: File.join(folder, state_name), + state_home: File.join(root, "state-home"), + project_name: "demo", + slug: "retry-task", + stage: stage, + now: Time.utc(2026, 7, 24, 12) + } + end + + def seed_crash_seam_request(fixture) + Hive::Daemon::AutoRetryHistory.new.reserve!( + task_folder: fixture[:folder], + task_id: 17, + slug: fixture[:slug], + stage: fixture[:stage], + marker_id: fixture[:attrs].fetch("marker_id"), + reason: fixture[:attrs].fetch("reason"), + fingerprint: "health-before-crash", + attempt: 1, + probes: [], + now: fixture[:now] - 1 + ) + anchor = Time.now + FileUtils.touch(fixture[:log_file], mtime: anchor - 120) + FileUtils.touch(fixture[:state_file], mtime: anchor - 60) + FileUtils.touch( + File.join(fixture[:folder], "status.md"), + mtime: anchor + 60 + ) + Hive::Daemon::DispatchRequestQueue.write_request!( + project: fixture[:project_name], + slug: fixture[:slug], + argv: [ + "hive", "run", fixture[:slug], "--project", fixture[:project_name], + "--stage", fixture[:stage] + ], + requestor: "healer", + trigger: "recoverable_dependency_failure", + expected_cleared_marker_id: fixture[:attrs].fetch("marker_id"), + state_home: fixture[:state_home], + now: fixture[:now] + ) + end + + def write_status_hive(root, fixture, include_identity: false) + status_path = File.join(root, "status.json") + calls_path = File.join(root, "status-calls.log") + File.write(status_path, JSON.generate(status_payload(fixture))) + identity = include_identity ? JSON.generate(Hive::InvokedBinary.loaded_identity) : "{}" + hive = write_executable(File.join(root, "bin", "hive"), <<~SH) + #!/bin/sh + if [ "$1" = "status" ]; then + printf 'status\\n' >> #{Shellwords.escape(calls_path)} + exec cat #{Shellwords.escape(status_path)} + fi + if [ "$1" = "--install-fingerprint" ]; then + printf '%s\\n' '#{identity}' + exit 0 + fi + exit 2 + SH + [ hive, calls_path ] + end + + def status_payload(fixture) + { + "schema" => "hive-status", + "schema_version" => Hive::Schemas::SCHEMA_VERSIONS.fetch("hive-status"), + "ok" => true, + "projects" => [ + { + "name" => fixture[:project_name], + "legacy_stage_dirs" => [], + "tasks" => [ + { + "id" => 17, + "slug" => fixture[:slug], + "stage" => fixture[:stage], + "marker" => "error", + "attrs" => fixture[:attrs], + "folder" => fixture[:folder], + "state_file" => fixture[:state_file], + "action" => "error", + "blocked" => false + } + ] + } + ] + } + end + + def with_registered_project(fixture, &block) + entry = { + "name" => fixture[:project_name], + "path" => fixture[:project], + "hive_state_path" => fixture[:hive_state] + } + with_replaced_singleton_method( + Hive::Config, :find_project, + ->(name) { name == fixture[:project_name] ? entry : nil }, + &block + ) + end + + def pending_requests(fixture) + Hive::Daemon::DispatchRequestQueue.pending(state_home: fixture[:state_home]) + end + + def write_executable(path, content) + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, content) + File.chmod(0o755, path) + path + end + + def read_events(folder) + File.readlines(File.join(folder, "events.jsonl"), chomp: true).map do |line| + JSON.parse(line) + end + end +end diff --git a/test/integration/daemon_command_test.rb b/test/integration/daemon_command_test.rb index a3c43fef..b371763c 100644 --- a/test/integration/daemon_command_test.rb +++ b/test/integration/daemon_command_test.rb @@ -3,6 +3,7 @@ require "json" require "open3" require "tmpdir" require "rbconfig" +require "hive/daemon/dispatch_request_queue" # Integration test for `hive daemon` subcommands. Uses real bin/hive # subprocesses against a temporary HIVE_HOME so PID file / log file diff --git a/test/integration/markers_command_test.rb b/test/integration/markers_command_test.rb index 19e14bfe..8aa68ff0 100644 --- a/test/integration/markers_command_test.rb +++ b/test/integration/markers_command_test.rb @@ -206,6 +206,62 @@ class MarkersCommandTest < Minitest::Test end end + def test_successful_manual_error_clear_records_reset_identity + with_tmp_global_config do + with_tmp_git_repo do |dir| + _, folder, _slug = seed_review_task(dir, marker: :error) + state = File.join(folder, "task.md") + Hive::Markers.set(state, :error, reason: "implementer_failed", marker_id: "current-marker") + + capture_io do + Hive::Commands::Markers.new( + "clear", folder, name: "ERROR", match_attr: "marker_id=current-marker" + ).call + end + + event = File.readlines(File.join(folder, "events.jsonl"), chomp: true) + .map { |line| JSON.parse(line) } + .find { |row| row["event_type"] == "marker_cleared" } + assert_equal "manual", event.dig("details", "actor") + assert_equal "current-marker", event.dig("details", "marker_id") + assert_equal "implementer_failed", event.dig("details", "marker_reason") + end + end + end + + def test_manual_clear_restores_marker_when_required_reset_event_cannot_append + with_tmp_global_config do + with_tmp_git_repo do |dir| + _, folder, _slug = seed_review_task(dir, marker: :error) + state = File.join(folder, "task.md") + Hive::Markers.set( + state, :error, + reason: "implementer_failed", marker_id: "current-marker" + ) + original = File.binread(state) + + with_replaced_singleton_method( + Hive::Events, :emit_required, + ->(**) { raise Errno::ENOSPC, "events.jsonl" } + ) do + assert_raises(Hive::InternalError) do + capture_io do + Hive::Commands::Markers.new( + "clear", folder, name: "ERROR", + match_attr: "marker_id=current-marker" + ).call + end + end + end + + assert_equal original, File.binread(state) + marker = Hive::Markers.current(state) + assert_equal :error, marker.name + assert_equal "current-marker", marker.attrs["marker_id"] + end + end + end + # ── Allowlist enforcement ────────────────────────────────────────────── def test_rejects_unknown_marker_name diff --git a/test/unit/bot/brainstorm_parser_test.rb b/test/unit/bot/brainstorm_parser_test.rb index 3f342cbd..16617f79 100644 --- a/test/unit/bot/brainstorm_parser_test.rb +++ b/test/unit/bot/brainstorm_parser_test.rb @@ -287,4 +287,17 @@ class HiveBotBrainstormParserTest < Minitest::Test assert_nil questions[0].answer, "Q1 with no A line must stay unanswered" assert_equal "yes", questions[1].answer end + + def test_answered_questions_returns_only_user_completed_slots + questions = Hive::Bot::BrainstormParser.parse_text(<<~MARKDOWN) + ## Round 1 + ### Q1. First? + ### A1. + yes + ### Q2. Second? + ### A2. + MARKDOWN + + assert_equal [ 1 ], Hive::Bot::BrainstormParser.answered_questions(questions).map(&:n) + end end diff --git a/test/unit/claude_launcher_test.rb b/test/unit/claude_launcher_test.rb index cb83b392..e98b90a1 100644 --- a/test/unit/claude_launcher_test.rb +++ b/test/unit/claude_launcher_test.rb @@ -1600,4 +1600,10 @@ class ClaudeLauncherTest < Minitest::Test assert_match(/example-task/, err) assert_match(/cleanup/, err) end + + def test_readiness_self_check_uses_production_detector + assert Hive::ClaudeLauncher.readiness_self_check + assert_equal File.expand_path("../../lib/hive/scripts/interactive_claude_wrapper.sh", __dir__), + Hive::ClaudeLauncher.interactive_wrapper_path + end end diff --git a/test/unit/commands/doctor_test.rb b/test/unit/commands/doctor_test.rb index 946169f2..766bb1aa 100644 --- a/test/unit/commands/doctor_test.rb +++ b/test/unit/commands/doctor_test.rb @@ -853,4 +853,133 @@ class HiveCommandsDoctorTest < Minitest::Test refute doctor.send(:legacy_brainstorm_runtime_present?) end end + + def test_required_checks_returns_structured_health_without_rendering + with_fake_home do |home| + install_brainstorm_and_plan_skills(home) + output = StringIO.new + doctor = Hive::Commands::Doctor.new( + config: base_config( + "brainstorm" => { "agent" => "claude", "skill" => "/x" }, + "plan" => { "agent" => "claude", "skill" => "/x" } + ), + project_root: nil, + output: output, + agent_health_checker: lambda { |profile| + [ "present", "#{profile.name} healthy" ] + } + ) + + result = doctor.required_checks + + assert result[:healthy] + assert_equal [ "claude" ], + result[:rows].select { |row| row[:kind] == "agent" }.map { |row| row[:agent] } + assert_equal %w[brainstorm plan], + result[:rows].select { |row| row[:kind] == "stage" }.map { |row| row[:stage] } + assert_empty output.string + end + end + + def test_required_checks_rejects_agent_below_configured_minimum + with_fake_home do |home| + install_brainstorm_and_plan_skills(home) + claude = File.join(home, "bin", "claude") + write_file(claude, "#!/bin/sh\necho 'Claude 2.1.0'\n") + File.chmod(0o755, claude) + cfg = base_config( + "brainstorm" => { "agent" => "claude", "skill" => "/x" }, + "plan" => { "agent" => "claude", "skill" => "/x" }, + "agents" => { + "claude" => { + "bin" => claude, + "env_override" => "HIVE_CLAUDE_BIN", + "min_version" => "9.0.0" + } + } + ) + + result = Hive::Commands::Doctor.new( + config: cfg, project_root: nil + ).required_checks + + refute result[:healthy] + agent = result[:rows].find { |row| row[:kind] == "agent" } + assert_equal "version_too_old", agent[:status] + assert_match(/below minimum 9\.0\.0/, agent[:message]) + ensure + Hive::AgentProfile.reset_version_cache! + end + end + + def test_required_checks_rejects_missing_agent_binary + with_fake_home do |home| + install_brainstorm_and_plan_skills(home) + missing = File.join(home, "bin", "missing-claude") + cfg = base_config( + "brainstorm" => { "agent" => "claude", "skill" => "/x" }, + "plan" => { "agent" => "claude", "skill" => "/x" }, + "agents" => { + "claude" => { + "bin" => missing, + "env_override" => "HIVE_CLAUDE_BIN", + "min_version" => "2.1.118" + } + } + ) + + result = Hive::Commands::Doctor.new( + config: cfg, project_root: nil + ).required_checks + + refute result[:healthy] + agent = result[:rows].find { |row| row[:kind] == "agent" } + assert_equal "missing", agent[:status] + assert_match(/binary not runnable/, agent[:message]) + ensure + Hive::AgentProfile.reset_version_cache! + end + end + + def test_required_checks_runs_agent_preflight + with_fake_home do |home| + write_file("#{home}/.pi/agent/skills/ce-brainstorm/SKILL.md") + write_file("#{home}/.pi/agent/skills/wiki-plan/SKILL.md") + pi = File.join(home, "bin", "pi") + write_file(pi, "#!/bin/sh\necho 'pi 0.70.2'\n") + File.chmod(0o755, pi) + pi_roles = Hive::Config::ROLE_AGENT_PATHS.each_with_object({}) do |path, config| + cursor = config + path.each_with_index do |part, index| + cursor[part] ||= index == path.length - 1 ? "pi" : {} + cursor = cursor[part] unless index == path.length - 1 + end + end + cfg = base_config(pi_roles).merge( + "agents" => { + "pi" => { + "bin" => pi, + "env_override" => "HIVE_PI_BIN", + "min_version" => "0.70.2" + } + }, + "review" => pi_roles.fetch("review").merge( + "ci" => pi_roles.dig("review", "ci").merge("command" => nil), + "browser_test" => pi_roles.dig("review", "browser_test").merge("enabled" => false) + ) + ) + + result = Hive::Commands::Doctor.new( + config: cfg, project_root: nil + ).required_checks + + refute result[:healthy] + agent = result[:rows].find { |row| row[:kind] == "agent" } + assert_equal "pi", agent[:agent] + assert_equal "missing", agent[:status] + assert_match(/no provider configured|not found/, agent[:message]) + ensure + Hive::AgentProfile.reset_version_cache! + end + end end diff --git a/test/unit/config_test.rb b/test/unit/config_test.rb index f9b9a92d..7c15a62f 100644 --- a/test/unit/config_test.rb +++ b/test/unit/config_test.rb @@ -2784,6 +2784,7 @@ class ConfigTest < Minitest::Test assert_equal 50, cfg.dig("daemon", "max_runs_per_day_per_project") assert_equal 60, cfg.dig("daemon", "transient_retry_backoff_sec") assert_equal 600, cfg.dig("daemon", "shutdown_grace_sec") + assert_equal true, cfg.dig("daemon", "auto_retry", "enabled") # R-02 per-child timeout knobs. assert_equal 0, cfg.dig("daemon", "child_timeout_sec") assert_equal 30, cfg.dig("daemon", "child_kill_grace_sec") @@ -2896,6 +2897,34 @@ class ConfigTest < Minitest::Test end end + def test_load_honors_and_validates_daemon_auto_retry_kill_switch + with_tmp_dir do |dir| + FileUtils.mkdir_p(File.join(dir, ".hive-state")) + path = File.join(dir, ".hive-state", "config.yml") + File.write(path, <<~YAML) + daemon: + auto_retry: + enabled: false + YAML + assert_equal false, Hive::Config.load(dir).dig("daemon", "auto_retry", "enabled") + + File.write(path, <<~YAML) + daemon: + auto_retry: + enabled: "no" + YAML + error = assert_raises(Hive::ConfigError) { Hive::Config.load(dir) } + assert_match(/daemon\.auto_retry\.enabled.*must be a boolean/, error.message) + + File.write(path, <<~YAML) + daemon: + auto_retry: enabled + YAML + error = assert_raises(Hive::ConfigError) { Hive::Config.load(dir) } + assert_match(/daemon\.auto_retry.*must be a Hash/, error.message) + end + end + def test_load_rejects_too_small_daemon_poll_interval with_tmp_dir do |dir| FileUtils.mkdir_p(File.join(dir, ".hive-state")) @@ -3020,12 +3049,15 @@ class ConfigTest < Minitest::Test fast_poll_sec: 2 max_concurrent_runs: 8 log_max_bytes: 524288 + auto_retry: + enabled: false YAML cfg = Hive::Config.load_global_daemon assert_equal 60, cfg["poll_interval_sec"] assert_equal 2, cfg["fast_poll_sec"] assert_equal 8, cfg["max_concurrent_runs"] assert_equal 524_288, cfg["log_max_bytes"] + assert_equal false, cfg.dig("auto_retry", "enabled") # Unspecified keys still pull from defaults assert_equal 50, cfg["max_runs_per_day_per_project"] assert_equal 3, cfg["max_concurrent_per_project"] diff --git a/test/unit/daemon/auto_retry_history_test.rb b/test/unit/daemon/auto_retry_history_test.rb new file mode 100644 index 00000000..37167830 --- /dev/null +++ b/test/unit/daemon/auto_retry_history_test.rb @@ -0,0 +1,130 @@ +require "test_helper" +require "hive/daemon/auto_retry_history" + +class AutoRetryHistoryTest < Minitest::Test + include HiveTestHelper + + def test_reservations_survive_reconstruction_and_exhaust_after_two + with_tmp_dir do |dir| + history = Hive::Daemon::AutoRetryHistory.new + now = Time.utc(2026, 7, 24, 12) + assert history.reserve!(**reservation_args(dir, attempt: 1, fingerprint: "health-a", now: now)) + assert history.reserve!(**reservation_args(dir, attempt: 2, fingerprint: "health-b", now: now + 1800)) + + decision = Hive::Daemon::AutoRetryHistory.new.assess( + task_folder: dir, reason: "implementer_failed", + fingerprint: "health-c", now: now + 3600 + ) + refute decision.eligible + assert_equal "retry_exhausted", decision.action + end + end + + def test_second_attempt_requires_changed_fingerprint_and_backoff + with_tmp_dir do |dir| + history = Hive::Daemon::AutoRetryHistory.new + now = Time.utc(2026, 7, 24, 12) + history.reserve!(**reservation_args(dir, attempt: 1, fingerprint: "same", now: now)) + + unchanged = history.assess(task_folder: dir, reason: "implementer_failed", + fingerprint: "same", now: now + 3600) + assert_equal "health_unchanged", unchanged.action + + early = history.assess(task_folder: dir, reason: "implementer_failed", + fingerprint: "changed", now: now + 1799) + assert_equal "retry_backoff", early.action + + ready = history.assess(task_folder: dir, reason: "implementer_failed", + fingerprint: "changed", now: now + 1800) + assert ready.eligible + assert_equal 2, ready.attempt + + assert_equal "retry_probe_due", + history.budget(task_folder: dir, reason: "implementer_failed", now: now + 1800).action + end + end + + def test_manual_clear_resets_only_matching_reason + with_tmp_dir do |dir| + history = Hive::Daemon::AutoRetryHistory.new + now = Time.utc(2026, 7, 24, 12) + history.reserve!(**reservation_args(dir, attempt: 1, fingerprint: "old", now: now)) + Hive::Events.emit( + task_folder: dir, slug: "retry-task", stage: "4-execute", + event_type: :marker_cleared, + details: { + actor: "manual", marker_reason: "implementer_failed", + marker_id: "marker-1", timestamp: (now + 1).iso8601 + } + ) + + decision = history.assess(task_folder: dir, reason: "implementer_failed", + fingerprint: "old", now: now + 2) + assert decision.eligible + assert_equal 1, decision.attempt + end + end + + def test_malformed_history_fails_closed + with_tmp_dir do |dir| + File.write(File.join(dir, "events.jsonl"), "{\"event_type\":") + decision = Hive::Daemon::AutoRetryHistory.new.assess( + task_folder: dir, reason: "implementer_failed", fingerprint: "x" + ) + refute decision.eligible + assert_equal "history_invalid", decision.action + end + end + + def test_invalid_reservation_timestamp_fails_closed_in_budget_and_assessment + with_tmp_dir do |dir| + details = { + "marker_reason" => "implementer_failed", + "attempt" => 1, + "health_fingerprint" => "health-a", + "timestamp" => "not-a-time" + } + File.write( + File.join(dir, "events.jsonl"), + "#{JSON.generate("event_type" => "auto_retry_reserved", "details" => details)}\n" + ) + history = Hive::Daemon::AutoRetryHistory.new + + assert_equal "history_invalid", + history.budget(task_folder: dir, reason: "implementer_failed").action + assert_equal "history_invalid", + history.assess( + task_folder: dir, reason: "implementer_failed", + fingerprint: "health-b" + ).action + end + end + + def test_reservation_rejects_non_json_probe_details + with_tmp_dir do |dir| + args = reservation_args( + dir, attempt: 1, fingerprint: "health-a", now: Time.now + ) + args[:probes] = [ Float::NAN ] + + refute Hive::Daemon::AutoRetryHistory.new.reserve!(**args) + end + end + + private + + def reservation_args(dir, attempt:, fingerprint:, now:) + { + task_folder: dir, + task_id: 42, + slug: "retry-task", + stage: "4-execute", + marker_id: "marker-#{attempt}", + reason: "implementer_failed", + fingerprint: fingerprint, + attempt: attempt, + probes: [], + now: now + } + end +end diff --git a/test/unit/daemon/auto_retry_policy_test.rb b/test/unit/daemon/auto_retry_policy_test.rb new file mode 100644 index 00000000..4579705c --- /dev/null +++ b/test/unit/daemon/auto_retry_policy_test.rb @@ -0,0 +1,352 @@ +require "test_helper" +require "hive/daemon/auto_retry_policy" +require "hive/daemon/status_consumer" + +class AutoRetryPolicyTest < Minitest::Test + include HiveTestHelper + + FakeTask = Struct.new( + :log_dir, :worktree_yml_path, :folder, :project_root, + keyword_init: true + ) + + def test_codex_auth_requires_production_execute_marker_and_auth_log + with_policy_fixture(stage: "4-execute", reason: "implementer_failed") do |fixture| + auth = "request failed with 401: Missing bearer or basic authentication in header" + fixture.write_log(auth) + fixture.marker_attrs.merge!("status" => "error", "message" => "exit_code=1") + fixture.rewrite_marker + + decision = fixture.policy.evaluate(fixture.row) + assert decision.eligible, decision.inspect + assert_equal "eligible", decision.action + + non_codex_policy = Hive::Daemon::AutoRetryPolicy.new( + task_builder: ->(_folder) { fixture.task }, + config_loader: ->(_root) { { "execute" => { "agent" => "claude" } } } + ) + denied = non_codex_policy.evaluate(fixture.row) + refute denied.eligible + assert_equal "codex_agent_mismatch", denied.action + + fixture.marker_attrs["message"] = auth + fixture.rewrite_marker + denied = fixture.policy.evaluate(fixture.row) + refute denied.eligible + assert_equal "codex_execute_marker_invalid", denied.action + end + end + + def test_stale_codex_auth_in_older_log_does_not_classify + with_policy_fixture(stage: "4-execute", reason: "implementer_failed") do |fixture| + auth = "401 Missing bearer or basic authentication in header" + fixture.write_log(auth, suffix: "old") + fixture.write_log("business logic failed", suffix: "new", newer: true) + fixture.marker_attrs.merge!("status" => "error", "message" => "exit_code=1") + fixture.rewrite_marker + + decision = fixture.policy.evaluate(fixture.row) + refute decision.eligible + assert_equal "codex_auth_signature_missing", decision.action + end + end + + def test_codex_auth_requires_current_agent_and_exact_episode_log + with_policy_fixture(stage: "4-execute", reason: "implementer_failed") do |fixture| + fixture.write_log("401 Missing bearer or basic authentication in header", suffix: "old") + fixture.marker_attrs.merge!( + "status" => "error", + "message" => "exit_code=1", + "log_file" => "execute-impl-current.log" + ) + fixture.rewrite_marker + + missing = fixture.policy.evaluate(fixture.row) + refute missing.eligible + assert_equal "codex_log_missing", missing.action + + fixture.marker_attrs["agent"] = "claude" + fixture.rewrite_marker + wrong_agent = fixture.policy.evaluate(fixture.row) + refute wrong_agent.eligible + assert_equal "codex_agent_mismatch", wrong_agent.action + end + end + + def test_claude_marker_requires_writer_attribution + with_policy_fixture(stage: "2-brainstorm", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + assert fixture.policy.evaluate(fixture.row).eligible + + fixture.marker_attrs.delete("writer") + fixture.rewrite_marker + decision = fixture.policy.evaluate(fixture.row) + refute decision.eligible + assert_equal "claude_marker_unattributed", decision.action + end + end + + def test_answered_brainstorm_and_substantive_plan_are_unsafe + with_policy_fixture(stage: "2-brainstorm", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + File.write(fixture.state_file, "## Round 1\n### Q1. Choose?\n### A1.\nUser answer\n") + fixture.rewrite_marker(append: true) + assert_equal "brainstorm_answered", fixture.policy.evaluate(fixture.row).action + end + + with_policy_fixture(stage: "3-plan", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + File.write(fixture.state_file, "# Plan\n\nBuild the retry subsystem.\n") + fixture.rewrite_marker(append: true) + assert_equal "plan_substantive", fixture.policy.evaluate(fixture.row).action + end + + with_policy_fixture(stage: "3-plan", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + File.write(fixture.state_file, "---\nowner: user\n---\n# Plan\n") + fixture.rewrite_marker(append: true) + assert_equal "plan_substantive", fixture.policy.evaluate(fixture.row).action + end + end + + def test_worktree_stages_require_a_completely_clean_git_worktree + with_policy_fixture(stage: "5-open-pr", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + assert fixture.policy.evaluate(fixture.row).eligible + + File.write(File.join(fixture.worktree, "dirty.txt"), "user work") + decision = fixture.policy.evaluate(fixture.row) + refute decision.eligible + assert_equal "worktree_dirty", decision.action + end + end + + def test_worktree_pointer_must_exist_be_contained_and_registered + with_policy_fixture(stage: "5-open-pr", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + File.unlink(fixture.task.worktree_yml_path) + assert_equal "worktree_pointer_missing", fixture.policy.evaluate(fixture.row).action + + File.write(fixture.task.worktree_yml_path, { "path" => fixture.project_root }.to_yaml) + assert_equal "worktree_pointer_invalid", fixture.policy.evaluate(fixture.row).action + + unregistered = File.join(File.dirname(fixture.worktree), "unregistered") + FileUtils.mkdir_p(unregistered) + run!("git", "-C", unregistered, "init", "-b", "main", "--quiet") + File.write(fixture.task.worktree_yml_path, { "path" => unregistered }.to_yaml) + assert_equal "worktree_unregistered", fixture.policy.evaluate(fixture.row).action + end + end + + def test_worktree_git_checks_have_a_hard_deadline + with_policy_fixture(stage: "5-open-pr", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + timed_out = Hive::Daemon::BoundedCommand::Result.new( + stdout: "", stderr: "", status: nil, timed_out: true + ) + policy = Hive::Daemon::AutoRetryPolicy.new( + task_builder: ->(_folder) { fixture.task }, + command_runner: ->(**) { timed_out } + ) + assert_equal "worktree_status_timeout", policy.evaluate(fixture.row).action + end + end + + def test_marker_identity_change_and_unknown_reason_fail_closed + with_policy_fixture(stage: "4-execute", reason: "unknown_failure") do |fixture| + decision = fixture.policy.evaluate(fixture.row) + refute decision.eligible + assert_equal "reason_not_allowlisted", decision.action + end + end + + def test_policy_errors_and_private_unknown_reason_fail_closed + with_policy_fixture(stage: "4-execute", reason: "implementer_failed") do |fixture| + policy = Hive::Daemon::AutoRetryPolicy.new( + task_builder: ->(_folder) { raise IOError, "task unreadable" } + ) + assert_equal "policy_error", policy.evaluate(fixture.row).action + + decision = fixture.policy.send( + :classify_reason, "future_reason", fixture.row, + FakeTask.new(log_dir: fixture.log_dir) + ) + assert_equal "reason_not_allowlisted", decision.action + end + end + + def test_codex_current_episode_and_readability_fail_closed + with_policy_fixture(stage: "4-execute", reason: "implementer_failed") do |fixture| + auth = "401 Missing bearer or basic authentication in header" + log = fixture.write_log(auth) + fixture.marker_attrs.merge!("status" => "error", "message" => "exit_code=1") + fixture.rewrite_marker + future = Time.now + 5 + File.utime(future, future, log) + assert_equal "codex_log_episode_ambiguous", + fixture.policy.evaluate(fixture.row).action + + with_replaced_singleton_method( + Hive::DiagnosticHelpers, :tail_file, + ->(_path) { raise IOError, "unreadable" } + ) do + assert_equal "codex_log_unreadable", + fixture.policy.evaluate(fixture.row).action + end + end + end + + def test_claude_unsupported_stage_marker_change_and_prior_success_are_unsafe + with_policy_fixture(stage: "1-inbox", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + assert_equal "claude_stage_unsupported", + fixture.policy.evaluate(fixture.row).action + end + + with_policy_fixture(stage: "2-brainstorm", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + Hive::Markers.set( + fixture.state_file, :error, + fixture.marker_attrs.merge("marker_id" => "replacement") + ) + assert_equal "marker_changed", fixture.policy.evaluate(fixture.row).action + end + + with_policy_fixture(stage: "2-brainstorm", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + File.write(fixture.state_file, "\n") + fixture.rewrite_marker(append: true) + assert_equal "terminal_success_present", + fixture.policy.evaluate(fixture.row).action + end + end + + def test_plan_scaffolding_passes_and_unknown_or_unreadable_stage_safety_denies + with_policy_fixture(stage: "3-plan", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + File.write(fixture.state_file, "# Plan\n\n") + fixture.rewrite_marker(append: true) + assert fixture.policy.evaluate(fixture.row).eligible + end + + with_policy_fixture(stage: "2-brainstorm", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + fixture.row.stage = "future-stage" + assert_equal "stage_safety_unknown", + fixture.policy.send(:safe_to_retry, fixture.row, Object.new).action + + with_replaced_singleton_method( + Hive::Markers, :current, ->(_path) { raise IOError, "unreadable" } + ) do + assert_equal "stage_state_unreadable", + fixture.policy.send(:safe_to_retry, fixture.row, Object.new).action + end + end + end + + def test_git_spawn_error_is_unsafe + with_policy_fixture(stage: "5-open-pr", reason: "claude_launch_failed") do |fixture| + fixture.attribute_claude! + policy = Hive::Daemon::AutoRetryPolicy.new( + task_builder: ->(_folder) { fixture.task }, + command_runner: ->(**) { raise Errno::ENOENT, "git" } + ) + assert_equal "worktree_status_failed", policy.evaluate(fixture.row).action + end + end + + private + + Fixture = Struct.new( + :dir, :project_root, :worktree, :task, :state_file, :log_dir, + :marker_attrs, :row, :policy, + keyword_init: true + ) do + def write_log(text, suffix: "current", newer: false) + path = File.join(log_dir, "execute-impl-#{suffix}.log") + File.write(path, text) + time = Time.now - (newer ? 1 : 2) + File.utime(time, time, path) + File.utime(Time.now, Time.now, state_file) + marker_attrs["agent"] = "codex" + marker_attrs["log_file"] = File.basename(path) + path + end + + def rewrite_marker(append: false) + if append + File.open(state_file, "a") do |file| + file.puts Hive::Markers.build_marker("ERROR", marker_attrs) + end + else + Hive::Markers.set(state_file, :error, marker_attrs) + end + row.marker_attrs = marker_attrs + end + + def attribute_claude! + marker_attrs.merge!( + "writer" => Hive::Daemon::AutoRetryPolicy::CLAUDE_MARKER_WRITER, + "exception_class" => "Hive::AgentError", + "message" => "launcher failed" + ) + rewrite_marker + end + end + + def with_policy_fixture(stage:, reason:, worktree: nil) + with_tmp_dir do |dir| + project_root = File.join(dir, "project") + worktree_root = File.join(dir, "worktrees") + FileUtils.mkdir_p(File.join(project_root, ".hive-state")) + File.write( + File.join(project_root, ".hive-state", "config.yml"), + { + "worktree_root" => worktree_root, + "execute" => { "agent" => "codex" } + }.to_yaml + ) + run!("git", "-C", project_root, "init", "-b", "main", "--quiet") + run!("git", "-C", project_root, "config", "user.email", "test@example.com") + run!("git", "-C", project_root, "config", "user.name", "Test") + File.write(File.join(project_root, "README.md"), "fixture\n") + run!("git", "-C", project_root, "add", "README.md") + run!("git", "-C", project_root, "commit", "-m", "fixture", "--quiet") + + folder = File.join(project_root, ".hive-state", "stages", stage, "retry-task") + FileUtils.mkdir_p(folder) + if Hive::Daemon::AutoRetryPolicy::WORKTREE_STAGES.include?(stage) + worktree ||= File.join(worktree_root, "retry-task") + run!("git", "-C", project_root, "worktree", "add", "--quiet", + "-b", "retry-task", worktree) + File.write(File.join(folder, "worktree.yml"), { "path" => worktree }.to_yaml) + end + state_file = File.join(folder, stage == "3-plan" ? "plan.md" : "task.md") + log_dir = File.join(project_root, ".hive-state", "logs", "retry-task") + FileUtils.mkdir_p(log_dir) + marker_attrs = { + "reason" => reason, + "marker_id" => "marker-current" + } + Hive::Markers.set(state_file, :error, marker_attrs) + row = Hive::Daemon::StatusConsumer::Row.new( + project: "project", slug: "retry-task", stage: stage, + marker: "error", marker_attrs: marker_attrs, folder: folder, + state_file: state_file, action: "error" + ) + task = FakeTask.new( + log_dir: log_dir, + worktree_yml_path: File.join(folder, "worktree.yml"), + folder: folder, + project_root: project_root + ) + policy = Hive::Daemon::AutoRetryPolicy.new(task_builder: ->(_folder) { task }) + fixture = Fixture.new( + dir: folder, project_root: project_root, worktree: worktree, task: task, + state_file: state_file, log_dir: log_dir, + marker_attrs: marker_attrs, row: row, policy: policy + ) + yield fixture + end + end +end diff --git a/test/unit/daemon/auto_retry_probe_test.rb b/test/unit/daemon/auto_retry_probe_test.rb new file mode 100644 index 00000000..969b98e0 --- /dev/null +++ b/test/unit/daemon/auto_retry_probe_test.rb @@ -0,0 +1,333 @@ +require "test_helper" +require "hive/config" +require "hive/daemon/auto_retry_probe" + +class AutoRetryProbeTest < Minitest::Test + include HiveTestHelper + + FakeDoctor = Struct.new(:result) do + def required_checks + result + end + end + + def test_codex_chain_requires_doctor_login_and_smoke_and_uses_isolated_cwd + with_tmp_dir do |project| + calls = [] + runner = lambda do |**args| + calls << args + probe_result(args[:name], healthy: true) + end + probe = build_probe(project, runner: runner) + + aggregate = probe.call(reason: "implementer_failed") + + assert aggregate.healthy + assert_equal %w[required_doctor codex_login_status codex_exec_smoke], + aggregate.probes.map(&:name) + smoke = calls.find { |call| call[:name] == "codex_exec_smoke" } + refute_equal project, smoke[:chdir] + refute smoke[:chdir].start_with?(project + File::SEPARATOR) + assert_equal 30, smoke[:timeout_sec] + assert_equal "/fake/codex", smoke[:env]["HIVE_CODEX_BIN"] + assert_includes smoke[:argv], "--skip-git-repo-check" + assert_includes smoke[:argv], "read-only" + assert_includes smoke[:argv], "--ephemeral" + assert_includes smoke[:argv], "shell_environment_policy.inherit=none" + refute_includes smoke[:argv], "--ignore-user-config" + refute_includes smoke[:argv], "--dangerously-bypass-approvals-and-sandbox" + end + end + + def test_unhealthy_command_or_doctor_blocks_aggregate + with_tmp_dir do |project| + runner = lambda do |**args| + probe_result(args[:name], healthy: args[:name] != "codex_login_status", + exit_status: 1) + end + aggregate = build_probe(project, runner: runner).call(reason: "implementer_failed") + refute aggregate.healthy + refute aggregate.probes.find { |probe| probe.name == "codex_login_status" }.healthy + + doctor = FakeDoctor.new({ healthy: false, rows: [], error: "missing skill" }) + aggregate = build_probe(project, runner: runner, doctor: doctor) + .call(reason: "implementer_failed") + refute aggregate.healthy + assert_equal "missing skill", aggregate.probes.first.rationale + end + end + + def test_claude_chain_checks_wrapper_detector_identity_and_binary + with_tmp_dir do |project| + identity = Hive::InvokedBinary.loaded_identity + runner = lambda do |**args| + stdout = args[:name] == "hive_install_identity" ? JSON.generate(identity) : "Claude 2.1.999" + probe_result(args[:name], healthy: true, stdout: stdout) + end + + aggregate = build_probe(project, runner: runner).call(reason: "claude_launch_failed") + + assert aggregate.healthy, aggregate.probes.map(&:to_h).inspect + assert_equal %w[ + required_doctor claude_wrapper claude_readiness_detector + hive_install_identity claude_binary + ], aggregate.probes.map(&:name) + end + end + + def test_identity_mismatch_blocks_claude_recovery + with_tmp_dir do |project| + runner = lambda do |**args| + stdout = if args[:name] == "hive_install_identity" + JSON.generate("version" => Hive::VERSION, "fingerprint" => "other-install") + else + "" + end + probe_result(args[:name], healthy: true, stdout: stdout) + end + aggregate = build_probe(project, runner: runner).call(reason: "claude_launch_failed") + refute aggregate.healthy + identity = aggregate.probes.find { |probe| probe.name == "hive_install_identity" } + assert_equal "daemon and CLI installation identities differ", identity.rationale + end + end + + def test_probe_outputs_are_redacted_bounded_and_fingerprints_are_stable + with_tmp_dir do |project| + secret = "Authorization: Bearer abcdefghijklmnopqrstuvwxyz" + invocation = 0 + runner = lambda do |**args| + invocation += 1 + probe_result( + args[:name], healthy: true, + stdout: "#{secret}#{invocation}-#{"x" * 10_000}" + ) + end + probe = build_probe(project, runner: runner) + first = probe.call(reason: "implementer_failed") + second = probe.call(reason: "implementer_failed") + + assert_equal first.fingerprint, second.fingerprint + first.probes.each do |result| + refute_includes result.stdout, "abcdefghijklmnopqrstuvwxyz" + assert_operator result.stdout.bytesize, :<=, Hive::Daemon::AutoRetryProbe::MAX_CAPTURE_BYTES + end + end + end + + def test_required_skill_inventory_changes_the_cheap_fingerprint + with_tmp_dir do |project| + doctor = FakeDoctor.new( + { healthy: false, rows: [ { kind: "skill", name: "missing", status: "missing" } ] } + ) + before = build_probe(project, runner: ->(**args) { + probe_result(args[:name], healthy: true) + }, doctor: doctor).cheap_fingerprint(reason: "implementer_failed") + + doctor.result = { + healthy: true, + rows: [ { kind: "skill", name: "missing", status: "ok", path: "/skills/missing" } ] + } + after = build_probe(project, runner: ->(**args) { + probe_result(args[:name], healthy: true) + }, doctor: doctor).cheap_fingerprint(reason: "implementer_failed") + + refute_equal before, after + end + end + + def test_cheap_fingerprints_ignore_the_other_reasons_dependency_signals + with_tmp_dir do |project| + runner = ->(**args) { probe_result(args[:name], healthy: true) } + base_env = ENV.to_h.merge( + "HIVE_CODEX_BIN" => "/fake/codex-a", + "HIVE_CLAUDE_BIN" => "/fake/claude-a" + ) + base = build_probe(project, runner: runner, env: base_env) + codex_before = base.cheap_fingerprint(reason: "implementer_failed") + claude_before = base.cheap_fingerprint(reason: "claude_launch_failed") + + claude_changed = build_probe( + project, runner: runner, + env: base_env.merge("HIVE_CLAUDE_BIN" => "/fake/claude-b") + ) + assert_equal codex_before, + claude_changed.cheap_fingerprint(reason: "implementer_failed") + refute_equal claude_before, + claude_changed.cheap_fingerprint(reason: "claude_launch_failed") + + codex_changed = build_probe( + project, runner: runner, + env: base_env.merge("HIVE_CODEX_BIN" => "/fake/codex-b") + ) + refute_equal codex_before, + codex_changed.cheap_fingerprint(reason: "implementer_failed") + assert_equal claude_before, + codex_changed.cheap_fingerprint(reason: "claude_launch_failed") + end + end + + def test_real_runner_times_out_and_marks_result_unhealthy + with_tmp_dir do |project| + script = File.join(project, "hang") + File.write(script, "#!/bin/sh\nsleep 5\n") + File.chmod(0o755, script) + probe = build_probe(project, runner: nil) + result = probe.send( + :run_command, + name: "hang", argv: [ script ], env: ENV.to_h, + chdir: project, timeout_sec: 0.05 + ) + + refute result.healthy + assert result.timed_out + assert_includes result.rationale, "timed out" + end + end + + def test_real_runner_does_not_wait_for_descendant_that_inherits_output + with_tmp_dir do |project| + script = File.join(project, "orphan-output") + File.write(script, <<~RUBY) + #!/usr/bin/env ruby + spawn("sleep", "5") + puts "leader done" + exit! 0 + RUBY + File.chmod(0o755, script) + probe = build_probe(project, runner: nil) + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + result = probe.send( + :run_command, + name: "orphan-output", argv: [ script ], env: ENV.to_h, + chdir: project, timeout_sec: 1 + ) + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + + assert result.healthy + assert_operator elapsed, :<, 1 + end + end + + def test_default_doctor_and_doctor_failure_modes_are_structured + with_tmp_dir do |project| + probe = Hive::Daemon::AutoRetryProbe.new( + config: Hive::Config::DEFAULTS, + project_root: project, + command_runner: ->(**args) { probe_result(args[:name], healthy: true) } + ) + assert_equal "required_doctor", probe.send(:doctor_probe).name + + timed_out = build_probe( + project, runner: ->(**args) { probe_result(args[:name], healthy: true) }, + doctor: Object.new.tap do |doctor| + doctor.define_singleton_method(:required_checks) { raise Timeout::Error } + end + ).send(:doctor_probe) + assert timed_out.timed_out + + raised = build_probe( + project, runner: ->(**args) { probe_result(args[:name], healthy: true) }, + doctor: Object.new.tap do |doctor| + doctor.define_singleton_method(:required_checks) { raise "doctor exploded" } + end + ).send(:doctor_probe) + refute raised.healthy + assert_includes raised.rationale, "doctor exploded" + end + end + + def test_probe_setup_and_aggregate_errors_become_unhealthy_results + with_tmp_dir do |project| + raising_runner = ->(**) { raise Errno::ENOENT, "codex" } + setup = build_probe(project, runner: raising_runner) + .call(reason: "implementer_failed") + refute setup.healthy + assert_equal "codex_probe_setup", setup.probes.last.name + + nil_runner = ->(**) { nil } + aggregate = build_probe(project, runner: nil_runner) + .call(reason: "implementer_failed") + refute aggregate.healthy + assert_equal "probe_aggregate", aggregate.probes.first.name + end + end + + def test_claude_local_probe_failures_are_structured + with_tmp_dir do |project| + probe = build_probe( + project, + runner: ->(**args) { + probe_result(args[:name], healthy: true, stdout: "not-json") + } + ) + invalid = probe.send(:hive_identity_probe) + refute invalid.healthy + assert_includes invalid.rationale, "invalid CLI identity JSON" + + with_replaced_singleton_method( + Hive::ClaudeLauncher, :interactive_wrapper_path, + -> { raise Errno::EACCES, "wrapper" } + ) do + refute probe.send(:wrapper_probe).healthy + assert_equal [ "wrapper", "unresolved" ], probe.send(:wrapper_signal) + end + + with_replaced_singleton_method( + Hive::AgentProfiles, :lookup, ->(*) { raise KeyError, "profile" } + ) do + refute probe.send(:claude_binary_probe).healthy + end + end + end + + def test_command_spawn_errors_are_structured + with_tmp_dir do |project| + probe = build_probe(project, runner: nil) + missing = probe.send( + :run_command, name: "missing", argv: [ File.join(project, "missing") ], + env: ENV.to_h, chdir: project, timeout_sec: 1 + ) + refute missing.healthy + assert_includes missing.rationale, "failed to start" + end + end + + def test_unresolved_binary_and_cheap_fingerprint_error_have_stable_fallbacks + with_tmp_dir do |project| + probe = build_probe(project, runner: ->(**args) { + probe_result(args[:name], healthy: true) + }) + assert_equal "definitely-missing", + probe.send(:resolve_binary, "definitely-missing") + probe.define_singleton_method(:cheap_fingerprint) do |reason:| + raise TypeError, reason + end + fallback = probe.send(:safe_cheap_fingerprint, reason: "implementer_failed") + assert_match(/\A[0-9a-f]{64}\z/, fallback) + end + end + + private + + def build_probe(project, runner:, doctor: FakeDoctor.new({ healthy: true, rows: [] }), + env: ENV.to_h.merge("HIVE_CODEX_BIN" => "/fake/codex")) + kwargs = { + config: Hive::Config::DEFAULTS, + project_root: project, + hive_bin: File.expand_path("../../../bin/hive", __dir__), + env: env, + doctor_factory: -> { doctor } + } + kwargs[:command_runner] = runner if runner + Hive::Daemon::AutoRetryProbe.new(**kwargs) + end + + def probe_result(name, healthy:, stdout: "", stderr: "", exit_status: 0) + Hive::Daemon::AutoRetryProbe::ProbeResult.new( + name: name, healthy: healthy, exit_status: exit_status, + timed_out: false, duration_ms: 1, stdout: stdout, stderr: stderr, + rationale: healthy ? "command succeeded" : "command failed" + ) + end +end diff --git a/test/unit/daemon/auto_retry_test.rb b/test/unit/daemon/auto_retry_test.rb new file mode 100644 index 00000000..7f60dca2 --- /dev/null +++ b/test/unit/daemon/auto_retry_test.rb @@ -0,0 +1,504 @@ +require "test_helper" +require "hive/daemon/auto_retry" +require "hive/daemon/auto_retry_policy" +require "hive/daemon/auto_retry_probe" +require "hive/daemon/status_consumer" + +class AutoRetryTest < Minitest::Test + include HiveTestHelper + + FakePolicyDecision = Struct.new( + :eligible, :action, :rationale, :reason, :marker_id, :task, + keyword_init: true + ) + FakePolicy = Struct.new(:decision) do + def evaluate(_row) + decision + end + end + FakeProbe = Struct.new(:aggregate) do + def cheap_fingerprint(reason:) + raise "wrong reason" unless reason == aggregate.reason + + aggregate.cheap_fingerprint + end + + def call(reason:) + raise "wrong reason" unless reason == aggregate.reason + + aggregate + end + end + FailingQueue = Module.new do + def self.pending(**) + [] + end + + def self.write_request!(**) + raise Errno::ENOSPC + end + end + RacingMarkers = Module.new do + def self.clear_current(*, **) + false + end + end + ClaimedRaceQueue = Module.new do + def self.pending(**) + [] + end + + def self.write_request!(**) + "claimed-request" + end + + def self.remove_if_unclaimed(*, **) + false + end + end + FailingClearEvents = Module.new do + def self.emit_required(**) + nil + end + + def self.emit(**) + nil + end + end + FailedReservationHistory = Struct.new(:decision) do + def budget(**) + decision + end + + def assess(**) + decision + end + + def reserve!(**) + false + end + end + + def test_first_healthy_attempt_reserves_queues_and_guardedly_clears + with_fixture do |fixture| + retryer = fixture.retryer + result = retryer.tick([ fixture.row ], now: fixture.now).first + + assert_equal "retry_queued", result.action + assert_equal 1, result.attempt + assert_equal :none, Hive::Markers.current(fixture.state_file).name + request = Hive::Daemon::DispatchRequestQueue.pending(state_home: fixture.state_home).first + assert_equal [ + "hive", "run", fixture.row.slug, "--project", fixture.row.project, + "--stage", fixture.row.stage + ], request.argv + assert_equal "healer", request.requestor + assert_equal "marker-1", request.expected_cleared_marker_id + refute Hive::Daemon::DispatchRequestQueue.expired?( + request, now: fixture.now + 86_400 + ) + + events = read_events(fixture.folder) + event_types = events.map { |event| event["event_type"] } + assert_includes event_types, "auto_retry_reserved" + assert_includes event_types, "marker_cleared" + retry_events = events.select do |event| + %w[auto_retry_reserved marker_cleared].include?(event["event_type"]) + end + assert retry_events.all? { |event| event.dig("details", "task_id") == 42 } + end + end + + def test_second_attempt_requires_changed_health_and_backoff_and_then_exhausts + with_fixture do |fixture| + first = fixture.retryer.tick([ fixture.row ], now: fixture.now).first + assert_equal "retry_queued", first.action + fixture.restore_marker("marker-2") + + early = fixture.retryer(fingerprint: "health-b") + .tick([ fixture.row ], now: fixture.now + 1799).first + assert_equal "retry_backoff", early.action + + unchanged = fixture.retryer(fingerprint: "health-a") + .tick([ fixture.row ], now: fixture.now + 1800).first + assert_equal "health_unchanged", unchanged.action + + second = fixture.retryer(fingerprint: "health-b") + .tick([ fixture.row ], now: fixture.now + 1800).first + assert_equal "retry_queued", second.action + assert_equal 2, second.attempt + fixture.restore_marker("marker-3") + + exhausted = fixture.retryer(fingerprint: "health-c") + .tick([ fixture.row ], now: fixture.now + 3600).first + assert_equal "retry_exhausted", exhausted.action + assert_equal :error, Hive::Markers.current(fixture.state_file).name + end + end + + def test_marker_race_withdraws_unclaimed_request + with_fixture do |fixture| + result = fixture.retryer(markers: RacingMarkers) + .tick([ fixture.row ], now: fixture.now).first + + assert_equal "marker_race", result.action + assert_empty Hive::Daemon::DispatchRequestQueue.pending(state_home: fixture.state_home) + assert_equal :error, Hive::Markers.current(fixture.state_file).name + end + end + + def test_queue_failure_leaves_marker_and_consumes_fail_closed_reservation + with_fixture do |fixture| + result = fixture.retryer(request_queue: FailingQueue) + .tick([ fixture.row ], now: fixture.now).first + + assert_equal "queue_failed", result.action + assert_equal :error, Hive::Markers.current(fixture.state_file).name + assert_equal "retry_backoff", + Hive::Daemon::AutoRetryHistory.new.budget( + task_folder: fixture.folder, reason: fixture.reason, + now: fixture.now + 1 + ).action + end + end + + def test_clear_audit_failure_rolls_marker_back_and_does_not_report_success + with_fixture do |fixture| + result = fixture.retryer(events: FailingClearEvents) + .tick([ fixture.row ], now: fixture.now).first + + assert_equal "coordinator_error", result.action + assert_equal :error, Hive::Markers.current(fixture.state_file).name + assert_equal 1, + Hive::Daemon::DispatchRequestQueue.pending( + state_home: fixture.state_home + ).length + end + end + + def test_restart_resumes_preclear_guarded_request_without_duplicating_it + with_fixture do |fixture| + history = Hive::Daemon::AutoRetryHistory.new + assert history.reserve!( + task_folder: fixture.folder, task_id: fixture.row.id, + slug: fixture.row.slug, stage: fixture.row.stage, + marker_id: "marker-1", reason: fixture.reason, + fingerprint: "health-a", attempt: 1, probes: [], + now: fixture.now + ) + Hive::Daemon::DispatchRequestQueue.write_request!( + project: fixture.row.project, slug: fixture.row.slug, + argv: [ + "hive", "run", fixture.row.slug, "--project", fixture.row.project, + "--stage", fixture.row.stage + ], + requestor: "healer", + trigger: "recoverable_dependency_failure", + expected_cleared_marker_id: "marker-1", + request_id: "abc12345", + state_home: fixture.state_home, now: fixture.now + ) + + result = fixture.retryer(history: history) + .tick([ fixture.row ], now: fixture.now + 1).first + + assert_equal "retry_queued", result.action + assert_equal "abc12345", result.request_id + assert_equal :none, Hive::Markers.current(fixture.state_file).name + pending = Hive::Daemon::DispatchRequestQueue.pending( + state_home: fixture.state_home + ) + assert_equal [ "abc12345" ], pending.map(&:request_id) + end + end + + def test_dry_run_and_duplicate_rows_do_not_over_mutate + with_fixture do |fixture| + dry = fixture.retryer(dry_run: true) + .tick([ fixture.row, fixture.row ], now: fixture.now) + + assert_equal 1, dry.length + assert_equal "would_retry", dry.first.action + assert_equal :error, Hive::Markers.current(fixture.state_file).name + refute Dir.exist?(fixture.state_home) + assert_empty Hive::Daemon::DispatchRequestQueue.pending(state_home: fixture.state_home) + refute File.exist?(File.join(fixture.folder, "events.jsonl")) + refute File.exist?(File.join(fixture.folder, "status.md")) + end + end + + def test_probe_cache_reuses_unchanged_health_until_fallback_and_bypasses_on_signal_change + with_fixture do |fixture| + calls = 0 + cheap = "cheap-a" + factory = lambda do |_row, _decision| + probe = FakeProbe.new(self.class.aggregate(fixture.reason, "health-a")) + probe.define_singleton_method(:cheap_fingerprint) { |reason:| cheap } + probe.define_singleton_method(:call) do |reason:| + calls += 1 + aggregate + end + probe + end + retryer = Hive::Daemon::AutoRetry.new( + policy: fixture.policy, probe_factory: factory, + state_home: fixture.state_home, dry_run: true + ) + + retryer.tick([ fixture.row ], now: fixture.now) + retryer.tick([ fixture.row ], now: fixture.now + 300) + retryer.tick([ fixture.row ], now: fixture.now + 599) + assert_equal 1, calls + + retryer.tick([ fixture.row ], now: fixture.now + 600) + assert_equal 2, calls + + cheap = "cheap-b" + retryer.tick([ fixture.row ], now: fixture.now + 601) + assert_equal 3, calls + end + end + + def test_probe_cache_is_scoped_to_the_task_across_ticks + with_fixture do |fixture| + calls = 0 + factory = lambda do |_row, _decision| + probe = FakeProbe.new(self.class.aggregate(fixture.reason, "health-a")) + probe.define_singleton_method(:call) do |reason:| + calls += 1 + aggregate + end + probe + end + retryer = Hive::Daemon::AutoRetry.new( + policy: fixture.policy, probe_factory: factory, + state_home: fixture.state_home, dry_run: true + ) + other = Hive::Daemon::StatusConsumer::Row.new(**fixture.row.to_h.merge( + id: 43, + slug: "other-task", + folder: File.join(File.dirname(fixture.folder), "other-task") + )) + + retryer.tick([ fixture.row ], now: fixture.now) + retryer.tick([ other ], now: fixture.now + 1) + + assert_equal 2, calls + end + end + + def test_same_project_reason_checks_tick_cache_before_building_probe_runner + with_fixture do |fixture| + factory_calls = 0 + cheap_calls = 0 + probe_calls = 0 + factory = lambda do |_row, _decision| + factory_calls += 1 + probe = FakeProbe.new(self.class.aggregate(fixture.reason, "health-a")) + probe.define_singleton_method(:cheap_fingerprint) do |reason:| + cheap_calls += 1 + "cheap" + end + probe.define_singleton_method(:call) do |reason:| + probe_calls += 1 + aggregate + end + probe + end + retryer = Hive::Daemon::AutoRetry.new( + policy: fixture.policy, probe_factory: factory, + state_home: fixture.state_home, dry_run: true + ) + same_project_context = Hive::Daemon::StatusConsumer::Row.new(**fixture.row.to_h.merge( + id: 43, + slug: "other-task", + folder: File.join(File.dirname(fixture.folder), "other-task"), + marker_attrs: fixture.row.marker_attrs.merge("marker_id" => "marker-2") + )) + other_project_context = Hive::Daemon::StatusConsumer::Row.new(**fixture.row.to_h.merge( + project: "other-project", + id: 44, + slug: "third-task", + folder: File.join(File.dirname(fixture.folder), "third-task"), + marker_attrs: fixture.row.marker_attrs.merge("marker_id" => "marker-3") + )) + + retryer.tick( + [ fixture.row, same_project_context, other_project_context ], + now: fixture.now + ) + + assert_equal 2, factory_calls + assert_equal 2, cheap_calls + assert_equal 2, probe_calls + end + end + + def test_unchanged_negative_decisions_are_audited_once_per_ten_minutes + with_fixture do |fixture| + aggregate = self.class.aggregate(fixture.reason, "unhealthy") + aggregate.healthy = false + aggregate.probes.first.healthy = false + audits = [] + retryer = Hive::Daemon::AutoRetry.new( + policy: fixture.policy, + probe_factory: ->(_row, _decision) { FakeProbe.new(aggregate) }, + state_home: fixture.state_home, + dry_run: true, + audit: ->(**attrs) { audits << attrs } + ) + + retryer.tick([ fixture.row ], now: fixture.now) + retryer.tick([ fixture.row ], now: fixture.now + 599) + assert_equal 1, audits.length + + retryer.tick([ fixture.row ], now: fixture.now + 600) + assert_equal 2, audits.length + assert_equal 1, retryer.instance_variable_get(:@negative_audits).length + refute File.exist?(File.join(fixture.folder, "events.jsonl")) + end + end + + def test_policy_denial_is_audited_and_row_exceptions_are_isolated + with_fixture do |fixture| + audits = [] + denied = FakePolicyDecision.new( + eligible: false, action: "worktree_dirty", rationale: "unsafe" + ) + retryer = Hive::Daemon::AutoRetry.new( + policy: FakePolicy.new(denied), + probe_factory: ->(*) { raise "must not probe" }, + state_home: fixture.state_home, + audit: ->(**attrs) { audits << attrs } + ) + assert_equal "worktree_dirty", + retryer.tick([ fixture.row ], now: fixture.now).first.action + assert_equal "worktree_dirty", audits.first[:action] + + exploding = Object.new + exploding.define_singleton_method(:evaluate) { |_row| raise "policy exploded" } + retryer = Hive::Daemon::AutoRetry.new( + policy: exploding, + probe_factory: ->(*) { raise "must not probe" }, + state_home: fixture.state_home, + audit: ->(**attrs) { audits << attrs } + ) + result = retryer.tick([ fixture.row ], now: fixture.now).first + assert_equal "coordinator_error", result.action + assert_includes result.rationale, "policy exploded" + end + end + + def test_reservation_failure_and_claimed_marker_race_fail_closed + with_fixture do |fixture| + decision = Hive::Daemon::AutoRetryHistory::Decision.new( + eligible: true, attempt: 1, action: "retry_eligible", rationale: "ready" + ) + history = FailedReservationHistory.new(decision) + result = fixture.retryer(history: history) + .tick([ fixture.row ], now: fixture.now).first + assert_equal "reservation_failed", result.action + assert_equal :error, Hive::Markers.current(fixture.state_file).name + + result = fixture.retryer( + markers: RacingMarkers, request_queue: ClaimedRaceQueue + ).tick([ fixture.row ], now: fixture.now).first + assert_equal "marker_race", result.action + assert_equal "marker changed after request was claimed", result.rationale + end + end + + def test_audit_sink_argument_error_does_not_abort_dry_run + with_fixture do |fixture| + retryer = fixture.retryer( + dry_run: true, + audit: ->(**) { raise ArgumentError, "bad audit sink" } + ) + + assert_equal "would_retry", + retryer.tick([ fixture.row ], now: fixture.now).first.action + end + end + + private + + Fixture = Struct.new( + :folder, :state_file, :state_home, :row, :now, :reason, :policy, + keyword_init: true + ) do + def retryer(fingerprint: "health-a", markers: Hive::Markers, + request_queue: Hive::Daemon::DispatchRequestQueue, dry_run: false, + history: Hive::Daemon::AutoRetryHistory.new, audit: nil, + events: Hive::Events) + aggregate = AutoRetryTest.aggregate(reason, fingerprint) + Hive::Daemon::AutoRetry.new( + policy: policy, + probe_factory: ->(_row, _decision) { AutoRetryTest::FakeProbe.new(aggregate) }, + state_home: state_home, + markers: markers, + request_queue: request_queue, + events: events, + dry_run: dry_run, + history: history, + audit: audit + ) + end + + def restore_marker(marker_id) + attrs = { + "reason" => reason, "marker_id" => marker_id, + "message" => "dependency failed" + } + Hive::Markers.set(state_file, :error, attrs) + row.marker_attrs = attrs + policy.decision.marker_id = marker_id + end + end + + def with_fixture + with_tmp_dir do |dir| + folder = File.join(dir, "task") + state_home = File.join(dir, "state-home") + FileUtils.mkdir_p(folder) + state_file = File.join(folder, "execute.md") + reason = "implementer_failed" + marker_id = "marker-1" + attrs = { "reason" => reason, "marker_id" => marker_id, "message" => "dependency failed" } + Hive::Markers.set(state_file, :error, attrs) + row = Hive::Daemon::StatusConsumer::Row.new( + project: "demo", id: 42, slug: "retry-task", stage: "4-execute", + marker: "error", marker_attrs: attrs, folder: folder, + state_file: state_file, action: "error" + ) + decision = FakePolicyDecision.new( + eligible: true, action: "eligible", rationale: "safe", + reason: reason, marker_id: marker_id, task: Object.new + ) + fixture = Fixture.new( + folder: folder, state_file: state_file, state_home: state_home, + row: row, now: Time.utc(2026, 7, 24, 12), reason: reason, + policy: FakePolicy.new(decision) + ) + yield fixture + end + end + + def read_events(folder) + path = File.join(folder, "events.jsonl") + return [] unless File.exist?(path) + + File.readlines(path, chomp: true).map { |line| JSON.parse(line) } + end + + def self.aggregate(reason, fingerprint) + probe = Hive::Daemon::AutoRetryProbe::ProbeResult.new( + name: "required_doctor", healthy: true, exit_status: 0, + timed_out: false, duration_ms: 1, stdout: "", stderr: "", + rationale: "healthy" + ) + Hive::Daemon::AutoRetryProbe::Aggregate.new( + reason: reason, healthy: true, probes: [ probe ], + fingerprint: fingerprint, cheap_fingerprint: "cheap" + ) + end +end diff --git a/test/unit/daemon/bounded_command_test.rb b/test/unit/daemon/bounded_command_test.rb new file mode 100644 index 00000000..05b07f31 --- /dev/null +++ b/test/unit/daemon/bounded_command_test.rb @@ -0,0 +1,58 @@ +require "test_helper" +require "rbconfig" +require "hive/daemon/bounded_command" + +class BoundedCommandTest < Minitest::Test + include HiveTestHelper + + def test_noisy_timed_command_is_capped_while_it_runs + with_tmp_dir do |dir| + script = File.join(dir, "noisy.rb") + File.write(script, <<~RUBY) + STDOUT.sync = true + STDERR.sync = true + loop do + STDOUT.write("o" * 16_384) + STDERR.write("e" * 16_384) + end + RUBY + + result = Hive::Daemon::BoundedCommand.capture( + env: {}, + argv: [ RbConfig.ruby, script ], + chdir: dir, + timeout_sec: 0.1, + max_bytes: 1024 + ) + + assert result.timed_out + assert_equal 1024, result.stdout.bytesize + assert_equal 1024, result.stderr.bytesize + assert_empty Dir.glob(File.join(dir, "hive-command-*")) + end + end + + def test_leader_exit_is_not_held_open_by_descendant_output_handles + with_tmp_dir do |dir| + script = File.join(dir, "orphan.rb") + File.write(script, <<~RUBY) + spawn(#{RbConfig.ruby.inspect}, "-e", "sleep 0.5") + STDOUT.write("done") + RUBY + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + result = Hive::Daemon::BoundedCommand.capture( + env: {}, + argv: [ RbConfig.ruby, script ], + chdir: dir, + timeout_sec: 1, + max_bytes: 1024 + ) + + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + assert result.status.success? + assert_equal "done", result.stdout + assert_operator elapsed, :<, 1 + end + end +end diff --git a/test/unit/daemon/dispatch_request_queue_test.rb b/test/unit/daemon/dispatch_request_queue_test.rb index a3c6f5e7..1da64fd8 100644 --- a/test/unit/daemon/dispatch_request_queue_test.rb +++ b/test/unit/daemon/dispatch_request_queue_test.rb @@ -13,7 +13,7 @@ class HiveDaemonDispatchRequestQueueTest < Minitest::Test def write_request(state_home, request_id:, created_at:, argv: [ "hive", "run", "slug-x", "--json" ], project: "hive", slug: "slug-x", requestor: "bot", chat_id: 42, update_id: 99, trigger: "answer_complete", schema_version: Q::SCHEMA_VERSION, - schema: "hive-dispatch-request") + schema: "hive-dispatch-request", expected_cleared_marker_id: nil) dir = Q.directory(state_home: state_home) filename = Q.filename_for(created_at: created_at, request_id: request_id) path = File.join(dir, filename) @@ -30,6 +30,7 @@ class HiveDaemonDispatchRequestQueueTest < Minitest::Test "update_id" => update_id, "trigger" => trigger } + payload["expected_cleared_marker_id"] = expected_cleared_marker_id unless expected_cleared_marker_id.nil? File.write(path, JSON.generate(payload)) path end @@ -55,6 +56,46 @@ class HiveDaemonDispatchRequestQueueTest < Minitest::Test end end + def test_guarded_request_round_trips_and_does_not_expire + Dir.mktmpdir("hive-dispatch-queue") do |dir| + created_at = Time.utc(2026, 5, 28, 18, 0, 0) + Q.write_request!( + project: "hive", slug: "slug-x", + argv: [ "hive", "run", "slug-x" ], + requestor: "healer", + expected_cleared_marker_id: "marker-1", + state_home: dir, now: created_at + ) + + request = Q.pending(state_home: dir).first + assert_equal "marker-1", request.expected_cleared_marker_id + refute Q.expired?(request, now: created_at + 86_400) + end + end + + def test_pending_rejects_invalid_marker_guard + Dir.mktmpdir("hive-dispatch-queue") do |dir| + write_request( + dir, request_id: "BAD", created_at: Time.utc(2026, 5, 28, 18), + expected_cleared_marker_id: "../replacement" + ) + reasons = [] + assert_empty Q.pending( + state_home: dir, + bad_handler: ->(path:, reason:) { reasons << reason } + ) + assert_equal [ "invalid_expected_cleared_marker_id" ], reasons + assert_raises(ArgumentError) do + Q.write_request!( + project: "hive", slug: "slug-x", + argv: [ "hive", "run", "slug-x" ], + expected_cleared_marker_id: 123, + state_home: dir + ) + end + end + end + def test_pending_skips_malformed_json_via_bad_handler Dir.mktmpdir("hive-dispatch-queue") do |dir| good = write_request(dir, request_id: "OK", created_at: Time.utc(2026, 5, 28, 18, 0, 0), slug: "good") diff --git a/test/unit/daemon/dispatcher_test.rb b/test/unit/daemon/dispatcher_test.rb index d5de2782..f129e19b 100644 --- a/test/unit/daemon/dispatcher_test.rb +++ b/test/unit/daemon/dispatcher_test.rb @@ -210,18 +210,36 @@ class HiveDaemonDispatcherTest < Minitest::Test end end + class FakeAutoRetry + attr_reader :calls + + def initialize(error: nil) + @error = error + @calls = [] + end + + def tick(rows, now:) + @calls << { rows: rows, now: now } + raise @error if @error + + [] + end + end + # ── construction helpers ─────────────────────────────────────────────── def make_dispatcher(rows: [], dry_run: false, with_merge_watcher: false, with_patrol_scheduler: false, project_enabled: true, dispatch_state: nil, status_result: nil, dispatch_request_state_home: nil, dispatch_result_state_home: nil, - with_digest_scheduler: false, with_answer_digest_scheduler: false) + with_digest_scheduler: false, with_answer_digest_scheduler: false, + auto_retry: FakeAutoRetry.new, auto_retry_enabled: true) config = { "daemon" => { "edit_debounce_sec" => 30, "poll_interval_sec" => 30, - "shutdown_grace_sec" => 60 + "shutdown_grace_sec" => 60, + "auto_retry" => { "enabled" => auto_retry_enabled } } } controller = Hive::Daemon::ConcurrencyController.new( @@ -261,7 +279,8 @@ class HiveDaemonDispatcherTest < Minitest::Test answer_digest_scheduler: answer_digest_scheduler, dry_run: dry_run, dispatch_request_state_home: dispatch_request_state_home, - dispatch_result_state_home: dispatch_result_state_home + dispatch_result_state_home: dispatch_result_state_home, + auto_retry: auto_retry ) # Bypass the Hive::Config.find_project / Config.load lookup chain # for unit tests — stub the predicate directly. @@ -279,7 +298,7 @@ class HiveDaemonDispatcherTest < Minitest::Test def close; end end - def row(project: "p1", slug: "s1", stage: "1-inbox", marker: "waiting", + def row(project: "p1", id: 1, slug: "s1", stage: "1-inbox", marker: "waiting", action: "ready_to_brainstorm", command: "hive brainstorm s1", mtime: T0 - 600, claude_pid_alive: nil, live_task_lock: nil, state_file: nil, folder: nil, marker_attrs: {}, @@ -287,7 +306,7 @@ class HiveDaemonDispatcherTest < Minitest::Test blocked: false, workflow: nil) folder ||= make_existing_row_folder(project: project, stage: stage, slug: slug) Row.new( - project: project, slug: slug, stage: stage, workflow: workflow, marker: marker, + project: project, id: id, slug: slug, stage: stage, workflow: workflow, marker: marker, folder: folder, state_file: state_file || File.join(folder, "idea.md"), state_file_mtime: mtime, action: action, @@ -664,6 +683,86 @@ class HiveDaemonDispatcherTest < Minitest::Test "SIGHUP reload must push the reloaded digest config into the scheduler") end + def test_auto_retry_runs_on_status_rows_and_isolated_errors_do_not_stop_dispatch + candidate = row( + action: "ready_to_plan", + command: "hive plan s1 --from 2-brainstorm" + ) + auto_retry = FakeAutoRetry.new(error: RuntimeError.new("probe exploded")) + dispatcher, supervisor, _controller, logger = make_dispatcher( + rows: [ candidate ], auto_retry: auto_retry + ) + + dispatcher.tick(now: T0) + + assert_equal [ candidate ], auto_retry.calls.first[:rows] + assert_equal 1, supervisor.spawned.length + error = logger.events.find { |name, _attrs| name == :auto_retry_error } + assert_equal "coordinator_error", error.last[:action] + assert_includes error.last[:rationale], "probe exploded" + end + + def test_auto_retry_kill_switch_returns_before_coordinator + auto_retry = FakeAutoRetry.new + dispatcher, = make_dispatcher( + rows: [ row ], auto_retry: auto_retry, auto_retry_enabled: false + ) + + dispatcher.tick(now: T0) + + assert_empty auto_retry.calls + end + + def test_auto_retry_receives_only_enabled_nonlegacy_project_rows + enabled = row(project: "enabled", slug: "enabled-task") + disabled = row(project: "disabled", slug: "disabled-task") + legacy = row(project: "legacy", slug: "legacy-task") + projects = [ + Hive::Daemon::StatusConsumer::ProjectInfo.new( + name: "enabled", legacy_stage_dirs: [] + ), + Hive::Daemon::StatusConsumer::ProjectInfo.new( + name: "disabled", legacy_stage_dirs: [] + ), + Hive::Daemon::StatusConsumer::ProjectInfo.new( + name: "legacy", legacy_stage_dirs: [ { "stage_dir" => "1-input" } ] + ) + ] + status = Hive::Daemon::StatusConsumer::Result.new( + ok: true, rows: [ enabled, disabled, legacy ], projects: projects, error: nil + ) + auto_retry = FakeAutoRetry.new + dispatcher, = make_dispatcher( + status_result: status, auto_retry: auto_retry + ) + dispatcher.define_singleton_method(:project_enabled?) do |project| + project != "disabled" + end + + dispatcher.tick(now: T0) + + assert_equal [ enabled ], auto_retry.calls.first[:rows] + end + + def test_reload_rebuilds_noninjected_auto_retry_coordinator + dispatcher, = make_dispatcher + built = [] + dispatcher.instance_variable_set(:@auto_retry_injected, false) + dispatcher.instance_variable_set( + :@auto_retry_factory, + lambda { + instance = FakeAutoRetry.new + built << instance + instance + } + ) + + dispatcher.send(:reload_config!) + + assert_equal built.last, dispatcher.instance_variable_get(:@auto_retry) + assert_equal 1, built.length + end + def test_answer_digest_scheduler_dispatches_global_answer_digest_without_project_gate dispatcher, sup, _ctrl, logger, _mw, _patrol, _digest, answer_digest = make_dispatcher( rows: [], @@ -2538,7 +2637,7 @@ end Q = Hive::Daemon::DispatchRequestQueue def write_request_file(dir, slug:, request_id:, created_at: T0, argv: nil, project: "p1", - trigger: "answer_complete") + trigger: "answer_complete", expected_cleared_marker_id: nil) argv ||= [ "hive", "run", slug, "--json" ] path = File.join(Q.directory(state_home: dir), Q.filename_for(created_at: created_at, request_id: request_id)) payload = { @@ -2554,14 +2653,22 @@ end "update_id" => 99, "trigger" => trigger } + payload["expected_cleared_marker_id"] = expected_cleared_marker_id unless expected_cleared_marker_id.nil? File.write(path, JSON.generate(payload)) path end - def stub_find_project!(dispatcher, project_name) + def stub_find_project!(dispatcher, project_name, project_path: "/tmp/nonexistent", + hive_state_path: "/tmp/nonexistent/.hive-state") Hive::Config.singleton_class.alias_method(:__orig_find_project, :find_project) unless Hive::Config.singleton_class.method_defined?(:__orig_find_project) Hive::Config.define_singleton_method(:find_project) do |name| - name == project_name ? { "name" => project_name, "path" => "/tmp/nonexistent", "hive_state_path" => "/tmp/nonexistent/.hive-state" } : nil + if name == project_name + { + "name" => project_name, + "path" => project_path, + "hive_state_path" => hive_state_path + } + end end dispatcher end @@ -3183,6 +3290,123 @@ end end end + def test_dispatch_request_blocked_for_legacy_layout_project + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + project = Hive::Daemon::StatusConsumer::ProjectInfo.new( + name: "p1", legacy_stage_dirs: [ { "stage_dir" => "1-input" } ] + ) + status = Hive::Daemon::StatusConsumer::Result.new( + ok: true, rows: [], projects: [ project ], error: nil + ) + dispatcher, sup, _ctrl, logger, _mw = make_dispatcher( + status_result: status, dispatch_request_state_home: state_home + ) + write_request_file(state_home, slug: "s1", request_id: "LEGACY") + stub_find_project!(dispatcher, "p1") + begin + dispatcher.tick(now: T0) + + blocked = logger.events.find do |name, attrs| + name == :dispatch_request_blocked && + attrs[:request_id] == "LEGACY" + end + assert_equal "legacy_layout_detected", blocked.last[:reason] + assert_empty sup.spawned + assert_equal 1, Q.pending(state_home: state_home).length + ensure + restore_find_project! + end + end + end + + def test_guarded_retry_waits_for_exact_marker_clear_then_dispatches + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + project_root = File.join(state_home, "project") + hive_state = File.join(project_root, ".hive-state") + folder = File.join(hive_state, "stages", "4-execute", "s1") + FileUtils.mkdir_p(folder) + state_file = File.join(folder, "task.md") + Hive::Markers.set( + state_file, :error, + reason: "implementer_failed", marker_id: "marker-1" + ) + status_file = File.join(folder, "status.md") + File.write(status_file, "# Derived status\n") + FileUtils.touch(status_file, mtime: Time.now + 60) + dispatcher, sup, _ctrl, logger, _mw = make_dispatcher( + rows: [], dispatch_request_state_home: state_home + ) + write_request_file( + state_home, slug: "s1", request_id: "GUARDED", + argv: [ "hive", "run", "s1", "--project", "p1", "--stage", "4-execute" ], + expected_cleared_marker_id: "marker-1" + ) + stub_find_project!( + dispatcher, "p1", + project_path: project_root, hive_state_path: hive_state + ) + begin + dispatcher.tick(now: T0 + Q::EXPIRY_SEC + 1) + + assert_empty sup.spawned + assert_equal 1, Q.pending(state_home: state_home).length + blocked = logger.events.find do |name, attrs| + name == :dispatch_request_blocked && + attrs[:request_id] == "GUARDED" + end + assert_equal "marker_not_cleared", blocked.last[:reason] + + assert Hive::Markers.clear_current( + state_file, expected_name: :error, + match_attrs: { "marker_id" => "marker-1" } + ) + dispatcher.tick(now: T0 + Q::EXPIRY_SEC + 2) + assert_equal 1, sup.spawned.length + ensure + restore_find_project! + end + end + end + + def test_guarded_retry_rejects_replacement_terminal_marker + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + project_root = File.join(state_home, "project") + hive_state = File.join(project_root, ".hive-state") + folder = File.join(hive_state, "stages", "4-execute", "s1") + FileUtils.mkdir_p(folder) + state_file = File.join(folder, "task.md") + Hive::Markers.set( + state_file, :error, + reason: "implementer_failed", marker_id: "replacement" + ) + dispatcher, sup, _ctrl, logger, _mw = make_dispatcher( + rows: [], dispatch_request_state_home: state_home + ) + write_request_file( + state_home, slug: "s1", request_id: "REPLACED", + argv: [ "hive", "run", "s1", "--project", "p1", "--stage", "4-execute" ], + expected_cleared_marker_id: "marker-1" + ) + stub_find_project!( + dispatcher, "p1", + project_path: project_root, hive_state_path: hive_state + ) + begin + dispatcher.tick(now: T0) + + rejected = logger.events.find do |name, attrs| + name == :dispatch_request_rejected && + attrs[:request_id] == "REPLACED" + end + assert_equal "marker_guard_mismatch", rejected.last[:reason] + assert_empty sup.spawned + assert_empty Q.pending(state_home: state_home) + ensure + restore_find_project! + end + end + end + # R-01 from PR #241 ce-code-review: a spawn failure (Errno::EAGAIN # under fork-exhaustion, or any other StandardError raised by # dispatch_request!) must not abort the rest of the pending queue. diff --git a/test/unit/daemon/logger_test.rb b/test/unit/daemon/logger_test.rb index d3dddc1c..8b9ab48f 100644 --- a/test/unit/daemon/logger_test.rb +++ b/test/unit/daemon/logger_test.rb @@ -61,6 +61,32 @@ class HiveDaemonLoggerTest < Minitest::Test end end + def test_auto_retry_events_accept_structured_identity_and_probe_fields + with_log do |logger, path| + logger.event( + :auto_retry_action, + project: "hive", + task_slug: "retry-task", + stage: "4-execute", + marker_id: "marker-1", + marker_reason: "implementer_failed", + probes: [ { name: "codex_login_status", healthy: true } ], + health_fingerprint: "sha256", + attempt: 1, + action: "retry_queued", + rationale: "dependency recovered", + timestamp: "2026-07-24T12:00:00Z" + ) + logger.close + + record = JSON.parse(File.read(path)) + assert_equal "auto_retry_action", record["event"] + assert_equal "retry_queued", record["action"] + assert_equal true, record.dig("probes", 0, "healthy") + assert_equal "sha256", record["health_fingerprint"] + end + end + # ── closed event enum ───────────────────────────────────────────────── def test_unknown_event_raises_argument_error diff --git a/test/unit/daemon/status_consumer_test.rb b/test/unit/daemon/status_consumer_test.rb index a342b30a..67ed889e 100644 --- a/test/unit/daemon/status_consumer_test.rb +++ b/test/unit/daemon/status_consumer_test.rb @@ -38,10 +38,11 @@ class HiveDaemonStatusConsumerTest < Minitest::Test } end - def task_row(slug:, stage: "1-inbox", marker: "waiting", + def task_row(slug:, id: 7, stage: "1-inbox", marker: "waiting", action: "ready_to_brainstorm", command: "hive brainstorm slug", mtime: Time.now.utc.iso8601) { + "id" => id, "stage" => stage, "slug" => slug, "folder" => "/tmp/p/#{stage}/#{slug}", @@ -74,6 +75,7 @@ class HiveDaemonStatusConsumerTest < Minitest::Test assert_equal 1, result.rows.size row = result.rows.first assert_equal "writero", row.project + assert_equal 7, row.id assert_equal "fix-bug", row.slug assert_equal "ready_to_brainstorm", row.action assert_equal "hive brainstorm slug", row.suggested_command diff --git a/test/unit/events_test.rb b/test/unit/events_test.rb index 28ac2224..3b220690 100644 --- a/test/unit/events_test.rb +++ b/test/unit/events_test.rb @@ -39,6 +39,30 @@ class EventsTest < Minitest::Test end end + def test_emit_details_is_optional_and_bounded + with_tmp_dir do |dir| + legacy = Hive::Events.emit(task_folder: dir, slug: "event-test", stage: "4-execute", + event_type: :stage_enter) + detailed = Hive::Events.emit( + task_folder: dir, slug: "event-test", stage: "4-execute", + event_type: :auto_retry_decision, + details: { action: "parked", attempt: 1 } + ) + + assert_equal %w[ts slug stage agent event_type message], legacy.keys + assert_equal({ "action" => "parked", "attempt" => 1 }, detailed.fetch("details")) + assert_raises(ArgumentError) do + Hive::Events.emit(task_folder: dir, slug: "event-test", stage: "4-execute", + event_type: :auto_retry_decision, details: "not-an-object") + end + assert_raises(ArgumentError) do + Hive::Events.emit(task_folder: dir, slug: "event-test", stage: "4-execute", + event_type: :auto_retry_decision, + details: { payload: "x" * Hive::Events::MAX_DETAILS_BYTES }) + end + end + end + def test_unknown_event_type_raises with_tmp_dir do |dir| assert_raises(ArgumentError) do @@ -130,6 +154,27 @@ class EventsTest < Minitest::Test end end + def test_emit_required_propagates_append_failures + with_tmp_dir do |dir| + original_open = File.method(:open) + File.define_singleton_method(:open) do |path, *args, **kwargs, &block| + if path.to_s.end_with?("events.jsonl") + raise Errno::ENOSPC, "events.jsonl" + end + + original_open.call(path, *args, **kwargs, &block) + end + assert_raises(Errno::ENOSPC) do + Hive::Events.emit_required( + task_folder: dir, slug: "required-event", stage: "4-execute", + event_type: :auto_retry_reserved + ) + end + ensure + File.define_singleton_method(:open, original_open) if original_open + end + end + def test_concurrent_emits_keep_every_line_parseable with_tmp_dir do |dir| threads = 5.times.map do |idx| diff --git a/test/unit/invoked_binary_test.rb b/test/unit/invoked_binary_test.rb index c259e6d2..23e39fc9 100644 --- a/test/unit/invoked_binary_test.rb +++ b/test/unit/invoked_binary_test.rb @@ -92,4 +92,66 @@ class InvokedBinaryTest < Minitest::Test end end end + + def test_loaded_identity_is_stable_and_source_bound + first = Hive::InvokedBinary.loaded_identity + second = Hive::InvokedBinary.loaded_identity + + assert_equal Hive::VERSION, first.fetch("version") + assert_equal first, second + assert_match(/\A[0-9a-f]{64}\z/, first.fetch("source_digest")) + assert_match(/\A[0-9a-f]{64}\z/, first.fetch("fingerprint")) + end + + def test_loaded_identity_is_captured_lazily_and_only_once + original = Hive::InvokedBinary.loaded_identity + calls = 0 + Hive::InvokedBinary.instance_variable_set(:@loaded_identity, nil) + + with_replaced_singleton_method( + Hive::InvokedBinary, :capture_loaded_identity, + lambda { + calls += 1 + { "version" => "test", "source_digest" => "source", "fingerprint" => "fingerprint" } + } + ) do + assert_equal 0, calls + assert_equal "fingerprint", Hive::InvokedBinary.loaded_identity.fetch("fingerprint") + assert_equal "fingerprint", Hive::InvokedBinary.loaded_identity.fetch("fingerprint") + assert_equal 1, calls + end + ensure + Hive::InvokedBinary.instance_variable_set(:@loaded_identity, original.freeze) if original + end + + def test_capture_loaded_identity_fails_closed_when_source_cannot_be_resolved + with_replaced_singleton_method( + File, :realpath, ->(_path) { raise Errno::ENOENT, "source" } + ) do + identity = Hive::InvokedBinary.capture_loaded_identity(source: "/missing/hive.rb") + assert_equal Hive::VERSION, identity.fetch("version") + assert_nil identity.fetch("fingerprint") + end + end + + def test_source_manifest_detects_same_version_component_drift + with_tmp_dir do |root| + lib = File.join(root, "lib") + component = File.join(lib, "hive", "claude_launcher.rb") + FileUtils.mkdir_p(File.dirname(component)) + File.write(File.join(lib, "hive.rb"), "module Hive; VERSION = 'x'; end\n") + File.write(component, "OLD = true\n") + + before = Hive::InvokedBinary.capture_loaded_identity( + source: File.join(lib, "hive.rb") + ) + File.write(component, "NEW = true\n") + after = Hive::InvokedBinary.capture_loaded_identity( + source: File.join(lib, "hive.rb") + ) + + refute_equal before.fetch("source_digest"), after.fetch("source_digest") + refute_equal before.fetch("fingerprint"), after.fetch("fingerprint") + end + end end diff --git a/test/unit/markers_test.rb b/test/unit/markers_test.rb index 36c743a9..095cf2bd 100644 --- a/test/unit/markers_test.rb +++ b/test/unit/markers_test.rb @@ -195,6 +195,25 @@ class MarkersTest < Minitest::Test end end + def test_clear_current_rolls_back_when_required_side_effect_fails + with_tmp_dir do |dir| + file = File.join(dir, "x.md") + Hive::Markers.set(file, :error, reason: "boom", marker_id: "marker-1") + + assert_raises(IOError) do + Hive::Markers.clear_current( + file, + expected_name: :error, + match_attrs: { "marker_id" => "marker-1" } + ) { raise IOError, "audit failed" } + end + + marker = Hive::Markers.current(file) + assert_equal :error, marker.name + assert_equal "marker-1", marker.attrs["marker_id"] + end + end + def test_set_writes_attrs with_tmp_dir do |dir| file = File.join(dir, "x.md") diff --git a/test/unit/stages/execute_test.rb b/test/unit/stages/execute_test.rb index af52b588..fb947951 100644 --- a/test/unit/stages/execute_test.rb +++ b/test/unit/stages/execute_test.rb @@ -149,7 +149,11 @@ class HiveStagesExecuteTest < Minitest::Test write_plan(task) write_pointer(task, "path" => File.join(dir, "worktree"), "branch" => task.slug, "execute_base_head" => "base") git = FakeGit.new(head: "base", branch: task.slug, dirty: false, ancestor_result: true) - result = { status: :error, error_message: "exit_code=1 compile error" } + result = { + status: :error, + error_message: "exit_code=1 compile error", + log_file: File.join(dir, ".hive-state", "logs", task.slug, "execute-impl-current.log") + } run_result = with_fake_git_and_spawn(git, result: result) do Hive::Stages::Execute.run_pass(task, execute_cfg("codex"), File.join(dir, "worktree")) @@ -161,8 +165,9 @@ class HiveStagesExecuteTest < Minitest::Test assert_equal "implementer_failed", marker.attrs["reason"] assert_equal "error", marker.attrs["status"] assert_equal "exit_code=1 compile error", marker.attrs["message"] + assert_equal "codex", marker.attrs["agent"] + assert_equal "execute-impl-current.log", marker.attrs["log_file"] refute marker.attrs.key?("retry_after") - refute marker.attrs.key?("provider") end end @@ -176,7 +181,11 @@ class HiveStagesExecuteTest < Minitest::Test write_plan(task) write_pointer(task, "path" => File.join(dir, "worktree"), "branch" => task.slug, "execute_base_head" => "base") git = FakeGit.new(head: "base", branch: task.slug, dirty: false, ancestor_result: true) - result = { status: :timeout, error_message: "claude stop hook did not signal completion" } + result = { + status: :timeout, + error_message: "claude stop hook did not signal completion", + log_file: File.join(dir, ".hive-state", "logs", task.slug, "execute-impl-timeout.log") + } run_result = with_fake_git_and_spawn(git, result: result) do Hive::Stages::Execute.run_pass(task, execute_cfg("codex"), File.join(dir, "worktree")) @@ -188,8 +197,9 @@ class HiveStagesExecuteTest < Minitest::Test assert_equal "implementer_failed", marker.attrs["reason"] assert_equal "timeout", marker.attrs["status"] assert_equal "claude stop hook did not signal completion", marker.attrs["message"] + assert_equal "codex", marker.attrs["agent"] + assert_equal "execute-impl-timeout.log", marker.attrs["log_file"] refute marker.attrs.key?("retry_after") - refute marker.attrs.key?("provider") end end diff --git a/wiki/commands/daemon.md b/wiki/commands/daemon.md index 3397a3ec..be3d7a50 100644 --- a/wiki/commands/daemon.md +++ b/wiki/commands/daemon.md @@ -3,7 +3,7 @@ title: hive daemon type: command source: lib/hive/commands/daemon.rb, lib/hive/daemon/* created: 2026-05-06 -updated: 2026-06-18 +updated: 2026-07-24 tags: [command, daemon, automation, json] --- @@ -46,7 +46,53 @@ hive daemon queue [list | show | prune] [--json] | `install` | (Re)writes the platform-native unit file (`~/.config/systemd/user/hive-daemon.service` on Linux, `~/Library/LaunchAgents/local.hive-daemon.plist` on macOS) and starts/enables the service. Installers and agent-assisted setup run this by default so daemon autostart is global install-time infrastructure, independent of any project. Without `--force`, refuses to overwrite a pre-existing unit (preserving operator hand-edits); exit `64` (USAGE) with a message pointing at `--force` so automation can branch without clobbering local changes. With `--force`, saves the previous content to a timestamped `.bak-YYYYMMDDTHHMMSSZ` (rotated, never overwritten) via atomic write, then — only when an existing unit was actually overwritten (the `upgraded` outcome) — restarts the running daemon on Linux / unloads-then-loads on macOS so new `Environment=` lines take effect (a first-time `--force` install with no prior unit just starts/enables, no restart). A service-manager failure (systemctl reload/enable, or launchctl load rejecting the unit) exits `70` (SOFTWARE). A host with no systemd-user manager at all is different: the unit is still written, but autostart cannot be enabled, so it exits `0` with the `unsupported` outcome (and `target_path` set to the written unit) — a known-platform limitation, not a failure. With `--json`, every outcome (success and error) emits a `hive-daemon-install.v1` envelope. Units point at the user-facing wrapper path when installers provide it, so bash/Homebrew installs preserve the GEM_HOME/GEM_PATH wrapper across login/reboot; `hv` invocations remain valid when Apache Hive shadows `hive`. Use this after upgrading hive when the unit template has changed or when autostart needs repair. | | `enable` | Sets `daemon.enabled: true` in `/.hive-state/config.yml`. This enrolls a project for dispatch; it does not install, start, or autostart the global daemon service. Surgical line-level YAML editor (upsert) preserves comments, key order, and file-mode bits across enable/disable flips; rejects inline-flow `daemon: { ... }`, CRLF endings, and 4-space-indented children before any write. Atomic write goes via tempfile + `flock(LOCK_EX)` + `fsync` + rename; tempfile is ensure-cleaned on rename failure (ENOSPC / EACCES / EXDEV). Pre-flight (`preflight_targets`) validates every target before any write so `--all` cannot half-flip the registry on a bad middle project. Pass a registered project name OR `--all` (mutually exclusive — passing both raises USAGE 64). Exit 64 on missing/unknown target / not-initialised project / no registered projects. With `--json`, emits a `hive-daemon-enroll` envelope on success and an `EnrollErrorKind` JSON error envelope on failure (`missing_project` / `unknown_project` / `project_and_all` / `not_initialised` / `no_projects` / `config` / `internal`); YAML parse failures surface as `Hive::ConfigError` (exit 78). | | `disable` | Same shape as `enable`, sets `daemon.enabled: false`. The next dispatcher tick honours the change automatically (per-tick enable-cache invalidation); `hive daemon reload` is optional for instant pickup. | -| `queue` | Read-only inspection of the dispatch-request queue the bot/web producers and `3-plan` healer write and the daemon consumes. Runs in the CLI process (no daemon contact); reads the same `/dispatch_requests/` directory. Current pending request files use `hive-dispatch-request.v2`, whose `requestor` enum is `bot|healer`; older/wrong versions are reported as malformed and pruned like other bad files. `list` (default) prints each pending request with `request_id age project/slug verb` plus `[EXPIRED]` / `[NOT-ALLOWLISTED]` flags and any malformed files. `show ` dumps one request's full payload (errors with exit 1 if the id is unknown; missing id is a USAGE error). `prune` removes expired + malformed request files (the daemon also does this lazily on its own tick) and reports the count. With `--json`, emits a `hive-daemon-queue.v1` envelope (`action`, `requests[]`, `request`, `malformed[]`, `pruned_count`). Unknown actions, missing `show` request ids, and unexpected queue-command exceptions emit the schema's `ErrorPayload` arm with `ok:false`, `error_kind` (`unknown_action` / `missing_request_id` / `internal`), and `message` before exiting non-zero. Claimed in-flight requests (`*.json.claimed`) are intentionally not listed — they are daemon-managed; see [[modules/daemon]] §"At-most-once dispatch via atomic claim". | +| `queue` | Read-only inspection of the dispatch-request queue the bot/web producers and daemon healers write and the daemon consumes. Runs in the CLI process (no daemon contact); reads the same `/dispatch_requests/` directory. Current pending request files use strict `hive-dispatch-request.v3`; older/wrong versions are reported as malformed. v3 adds an internal marker guard for durable dependency retries while preserving `requestor=bot|healer`. `list` (default) prints each pending request with `request_id age project/slug verb` plus `[EXPIRED]` / `[NOT-ALLOWLISTED]` flags and any malformed files. `show ` dumps the public inspection fields. `prune` removes expired ordinary requests plus malformed files; guarded dependency requests do not expire before claim/rejection. With `--json`, emits a `hive-daemon-queue.v1` envelope. Claimed requests remain daemon-managed and hidden. | + +## Dependency auto-retry + +The daemon automatically revisits only two parked dependency markers: + +- a current execute `implementer_failed` episode whose marker is the real + `agent=codex, status=error, message=exit_code=N` execute shape, whose + configured execute profile is Codex, and whose marker-named `log_file` + contains both Codex auth signatures (401 and missing bearer/basic + authentication); +- a current `claude_launch_failed` marker stamped by the stage launch boundary. + +Retry still requires untouched/safe stage output, the required-agent/skill +doctor subset, and reason-specific dependency probes. Codex requires +`codex login status` plus an isolated, ephemeral, read-only `codex exec` smoke +test that opts out of the Git-repository check, uses the normal Codex user +configuration/provider selection, and disables model-generated shell +environment inheritance. +Claude requires the packaged interactive wrapper, readiness-detector canary, +matching daemon/CLI install identity, and a runnable Claude binary. + +Guarded requests resolve the task's authoritative workflow state file from the +queued `--stage`. A newer derived `status.md` therefore cannot make a live +failure look markerless. Automatic clear and its `marker_cleared` task event +are one fail-closed operation: an event-append failure restores the marker. + +There are at most two durable automatic reservations per task/reason. Attempt +two requires changed health and a 30-minute backoff. Exhaustion persists across +daemon restarts. Use the guarded manual fallback to reset that reason episode: + +```bash +hive markers clear --name ERROR \ + --match-attr marker_id= --project +hive run --project --stage +``` + +The global default-on kill switch is: + +```yaml +daemon: + auto_retry: + enabled: false +``` + +Send `hive daemon reload` after changing global config. See +[[modules/daemon]] for probe cadence and audit fields. ## Global Digest diff --git a/wiki/commands/doctor.md b/wiki/commands/doctor.md index 81cc8f8b..f7898aa6 100644 --- a/wiki/commands/doctor.md +++ b/wiki/commands/doctor.md @@ -3,7 +3,7 @@ title: hive doctor type: command source: lib/hive/commands/doctor.rb, lib/hive/skill_check.rb created: 2026-05-07 -updated: 2026-06-14 +updated: 2026-07-25 tags: [command, preflight, skills, tmux] --- @@ -37,6 +37,19 @@ Run from a hive-initialized project (loads `/.hive-state/config.yml`). Reviewer entries with `kind != "agent"` short-circuit to `:not_applicable` with a "kind '' is not 'agent'; doctor only checks agent-kind reviewers" message. `Hive::Config.validate_reviewers!` now validates `kind` against `agent`, `codex_review`, and `linter`; Doctor still only verifies slash-command/SKILL.md resolution for `agent` reviewers because `codex_review` uses Codex's built-in `review` subcommand and `linter` is intentionally rejected by reviewer dispatch with a pointer to `review.ci.command`. +## Daemon required subset + +`Doctor#required_checks` is the lightweight in-process subset used before every +dependency auto-retry. In addition to the stage/reviewer skill rows, it emits +one `kind: "agent"` row for each distinct enabled profile required by stage +roles and agent-backed reviewers. Each row resolves project profile overrides, +runs ` `, enforces `min_version`, and calls the profile +preflight (for example Pi provider authentication). Missing binaries, old +versions, failed preflights, and verifier failures make the aggregate +unhealthy. Disabled browser/triage roles and review CI without a command are +not treated as required. This subset does not render output and excludes +advisory QMD/tmux/runtime warnings. + ## Per-agent verifiers (`Hive::SkillCheck::*`) Encoded as the third return of `AgentProfile.new(skill_verifier:)`: @@ -86,7 +99,7 @@ Rescue scope is `StandardError` (with a `Errno::EPIPE` micro-rescue around `warn ## Tests -- `test/unit/commands/doctor_test.rb` — stage rows, reviewer happy path, mixed agents, empty/nil/absent reviewers, non-agent kinds, pi reviewer rows, QMD managed-binary and broken-binary rows, JSON envelope shape, long-label width, `attr_reader :rows` exposure. +- `test/unit/commands/doctor_test.rb` — stage rows, reviewer happy path, mixed agents, empty/nil/absent reviewers, non-agent kinds, pi reviewer rows, QMD managed-binary and broken-binary rows, JSON envelope shape, long-label width, `attr_reader :rows` exposure, and the daemon subset's required binary/minimum-version/preflight checks. - `test/unit/skill_check_test.rb` — per-agent verifier paths (including pi recursive walks, settings entries, manifest entries, global npm-root success/timeout handling, and glob-metacharacter rejection). - `test/integration/init_doctor_preflight_test.rb` — all-green silence, single-missing stderr warning, multi-missing including a reviewer row, init exit-code unchanged, preflight crash → bug-hint warning, config-load error → bug-hint warning. diff --git a/wiki/commands/markers.md b/wiki/commands/markers.md index ec76e1e4..8781bb01 100644 --- a/wiki/commands/markers.md +++ b/wiki/commands/markers.md @@ -3,7 +3,7 @@ title: hive markers type: command source: lib/hive/commands/markers.rb created: 2026-04-26 -updated: 2026-05-27 +updated: 2026-07-24 tags: [command, markers, recovery, json] --- @@ -41,7 +41,12 @@ Only recovery markers are clearable. Terminal-success markers (`REVIEW_COMPLETE` 3. Validate the requested `--name` against `Hive::Commands::Markers::ALLOWED_NAMES`. Anything else raises `Hive::WrongStage` (exit 4). 4. Read the current marker via `Hive::Markers.current(state_file)`. If the marker name does NOT match `--name`, raise `Hive::WrongStage` — refusing to silently clear a different state. 5. If `--match-attr` is present, require every supplied `KEY=VALUE` pair to match the current marker. Comma-separated pairs such as `reason=exit_code,exit_code=143` are all checked; any mismatch raises `Hive::WrongStage`. TUI ERROR recovery prefers generated `marker_id` attrs when available and uses observed reason/exit_code attrs for legacy rows. -6. Remove the marker line: `File.read` the body, `sub` out the exact `marker.raw` comment plus its trailing newline (if it sat alone on a line), then `Hive::Markers.write_atomic` the result. Surrounding prose, headings, and other markers stay untouched. +6. Under the marker lock, remove the marker and append the required + `marker_cleared` task event with task id/slug, cleared marker identity/reason, + and `actor=manual`. For dependency auto-retry, this event starts a fresh + durable attempt episode for that reason. If the event append fails, restore + the original state file and fail the command; a clear can no longer report + success without its reset history. 7. Record a `hive_commit` on the `hive/state` branch (`hive: / markers clear `). 8. Emit a stdout summary (or one-line `hive-markers-clear` JSON document with `--json`); print a `next: hive run ` hint to stderr. diff --git a/wiki/decisions.md b/wiki/decisions.md index 371a107c..5149464a 100644 --- a/wiki/decisions.md +++ b/wiki/decisions.md @@ -125,7 +125,7 @@ behavior is covered in [[commands/web]]. **Decision:** Eliminate the dual-writer for state-mutating `hive` verbs by routing all of them through a file-backed request queue at `/dispatch_requests/`. The bot stops being a subprocess caller for the allowlisted verb set; instead it writes a JSON request file via `Hive::Bot::DispatchRequestWriter.write!` (atomic via tmp + rename). Hivebox web later reused the same producer path for stage-run/recovery requests, and `Hive::Daemon::StaleAgentHealer` now enqueues `3-plan` reruns as `requestor=healer`. The daemon's tick loop scans the queue via `Hive::Daemon::DispatchRequestQueue.pending`, validates argv against `ALLOWED_VERBS` (and the request's slug against ADR-012's regex + project against a name regex), and dispatches through the same code path as auto-advance. The daemon's existing reap + `refresh_post_completion_mtime` then keeps the baseline current. -The queue schema is registered in `Hive::Schemas::SCHEMA_VERSIONS` and published under `schemas/` (per ADR-025 — every entry in SCHEMA_VERSIONS must have a corresponding schema file). `hive-dispatch-request.v1.json` was the original bot-only contract; current code writes `hive-dispatch-request.v2.json`, whose breaking change is the `requestor` enum gaining `healer` while preserving `bot` for Telegram and hivebox web. The live queue is strict-version-matched: mismatched versions are rejected as malformed/`unknown_schema_version`, so future queue-shape changes require a coordinated producer/daemon bump plus a new schema file before emission. +The queue schema is registered in `Hive::Schemas::SCHEMA_VERSIONS` and published under `schemas/` (per ADR-025 — every entry in SCHEMA_VERSIONS must have a corresponding schema file). `hive-dispatch-request.v1.json` was the original bot-only contract; v2 added `requestor=healer`; current v3 adds the optional `expected_cleared_marker_id` used by durable dependency retries. The live queue is strict-version-matched, so an older daemon rejects v3 instead of silently ignoring its dispatch guard. The allowlist is closed: `run develop brainstorm plan review open-pr artifacts finalize archive markers`. Adding a new state-mutating verb to the daemon requires updating `ALLOWED_VERBS` and the schema's `$defs.ALLOWED_VERBS` in lockstep — a unit test asserts cross-list equality. @@ -134,7 +134,7 @@ The allowlist is closed: `run develop brainstorm plan review open-pr artifacts f **Consequences:** - The daemon is now the SOLE process that spawns `hive run`-class children. The structural cause of the cross-process mtime-baseline bug is gone. - The queue dir is created with mode 0700: the producer/consumer authentication boundary is the filesystem-permissions invariant. The `requestor` field in each request is informational only; the daemon does NOT verify it against process credentials. Multi-user hosts therefore depend on per-user `~/.local/state/hive/` ownership, which is the existing operating-system assumption. -- A new request type expires after 600s. If the daemon is down for longer, queued requests are pruned on its next tick (logged as `dispatch_request_expired`). Operators restarting after extended outage see no automatic re-trigger. +- Ordinary requests expire after 600s. Marker-guarded dependency retry requests are the narrow exception: they remain durable until claimed or rejected because their marker was unavailable/replaced. - Telemetry: 5 daemon events (`dispatch_request_observed/dispatched/completed/rejected/blocked/expired`) provide a full lifecycle trace keyed by `request_id`. The bot continues to log via `:dispatched_command` with `via=queue` and `request_id` so the same correlation ID grep works across daemon.log and bot.log. - A per-iteration rescue in `process_dispatch_requests` ensures one request's `Process.spawn` failure (Errno::EAGAIN under fork-exhaustion etc.) does not abort the rest of the tick — the failing request's file stays on disk for the next tick to retry. - Notification preservation: the bot's "next question" message is now driven exclusively by the daemon's notification path (status-poll). A `notification_exactly_once` integration test pins the invariant. There is a slightly larger latency window (one tick) between agent completion and operator notification compared to the pre-refactor bot-side reaper, but the deduplication guarantee is stronger. diff --git a/wiki/log.d/20260724T120000Z-daemon-recoverable-marker-auto-retry.md b/wiki/log.d/20260724T120000Z-daemon-recoverable-marker-auto-retry.md new file mode 100644 index 00000000..49bd6295 --- /dev/null +++ b/wiki/log.d/20260724T120000Z-daemon-recoverable-marker-auto-retry.md @@ -0,0 +1,25 @@ +## [2026-07-24T12:00:00Z] daemon — add fail-closed dependency marker auto-retry + +**Action:** Added a dedicated, default-enabled daemon subsystem that +automatically retries exactly correlated Codex authentication failures and +launcher-attributed Claude failures after conservative stage-safety and +dependency-health gates pass. + +**Code:** +- `AutoRetryPolicy` recognizes the fixed marker/diagnostic contracts and + rejects ambiguous partial output or dirty/unresolvable worktrees. +- `AutoRetryProbe` shares the required doctor checks and runs bounded, + redacted Codex or Claude health chains in the daemon's resolved environment. +- `AutoRetryHistory` persists a two-attempt task/reason budget in + `events.jsonl`; attempt two requires changed health plus a 30-minute backoff, + and only a successful manual marker clear resets the episode. +- `AutoRetry` reserves, persists the normal same-stage queue request, then + clears only the current marker id; races withdraw only unclaimed requests. +- `daemon.auto_retry.enabled: false` is the global kill switch. Probe and + unchanged-negative fallbacks are both 600 seconds. + +**Validation:** +- Focused policy, probe, history, coordinator, config, dispatcher, logger, + marker, doctor, launcher, and queue tests. +- Hermetic Codex and Claude integration coverage in + `test/integration/daemon_auto_retry_test.rb`. diff --git a/wiki/log.d/20260725T013518Z-auto-retry-review-hardening.md b/wiki/log.d/20260725T013518Z-auto-retry-review-hardening.md new file mode 100644 index 00000000..6f7005e4 --- /dev/null +++ b/wiki/log.d/20260725T013518Z-auto-retry-review-hardening.md @@ -0,0 +1,37 @@ +# Auto-retry review hardening + +**Date:** 2026-07-25 + +**Action:** Closed the first review pass’s fail-closed and crash-durability +gaps in dependency auto-retry. Codex auth classification now starts from the +production execute marker shape and correlates it with auth-only log evidence. +Planning frontmatter is treated as content, and worktree stages require an +explicit contained, registered, clean pointer checked under a hard deadline. + +Dependency probes now use tempfile capture and process-group deadlines; the +Codex smoke is ephemeral/read-only, skips the Git repository prerequisite, and +does not expose the daemon environment to model-generated shell commands. +Required-skill inventory invalidates cache, cache age is no longer extended by +hits, volatile output is excluded from health identity, and daemon/CLI install +identity snapshots the loaded Ruby source manifest. + +Dispatch requests moved to strict `hive-dispatch-request.v3`. Automatic retry +requests carry `expected_cleared_marker_id`, survive ordinary queue expiry, +block while the exact marker remains, and reject replacement/unreadable state. +A restarted coordinator resumes the existing durable reservation/request +instead of duplicating it. Disabled and legacy-layout projects are gated before +both retry coordination and queued dispatch. + +Audit records now carry numeric task ids; dry-run avoids task event/status +mutation; expired negative-audit throttle keys are evicted. Required event +appends back retry reservations and manual retry-history resets, and a manual +clear restores the marker if its reset event cannot be persisted. + +**Tests:** Expanded unit and integration coverage for production marker/log +classification, registered worktree pointers, subprocess deadlines, safe Codex +argv, stable health fingerprints, skill/cache invalidation, pre/post-clear +queue durability, marker replacement, project gates, task ids, dry-run +immutability, event failure rollback, and loaded-source drift. + +**See:** [[modules/daemon]], [[state-model]], [[commands/daemon]], +[[commands/markers]], [[modules/events]], [[testing]] diff --git a/wiki/log.d/20260725T030000Z-auto-retry-review-pass-2.md b/wiki/log.d/20260725T030000Z-auto-retry-review-pass-2.md new file mode 100644 index 00000000..d191ab4a --- /dev/null +++ b/wiki/log.d/20260725T030000Z-auto-retry-review-pass-2.md @@ -0,0 +1,32 @@ +# Auto-retry review hardening, pass 2 + +**Date:** 2026-07-25 + +**Action:** Closed the second review pass's episode-correlation, crash-seam, +probe-parity, cache-scope, and audit-durability gaps. Execute failure markers +now identify their agent and exact implementer log; Codex recovery requires +the configured and recorded agent to be Codex. Guarded queue dispatch resolves +the authoritative workflow state file from the queued stage instead of the +newest Markdown file. + +The required doctor subset now verifies every enabled required agent binary, +minimum version, and profile preflight. Codex smoke tests retain production +user configuration. Cheap fingerprints contain only reason-relevant binary, +environment, wrapper, and install signals; task-scoped aggregates are reused +before constructing another runner. Subprocess capture continuously drains +pipes while retaining a fixed byte ceiling. + +Automatic marker clear now requires its paired task event and restores the +marker if that append fails. Loaded-source identity hashing is lazy for +ordinary CLI startup and explicitly captured when a daemon dispatcher is +constructed. Dead dry-run retry code was removed. + +**Tests:** Added the full `StatusConsumer` → dispatcher-tick → guarded queue +claim/spawn acceptance path, including a pre-clear crash reservation whose +newer `status.md` must not bypass the live marker. Expanded unit coverage for +exact Codex episode/agent attribution, required agent health, production Codex +argv, reason-specific fingerprints, task-scoped caches, bounded live output, +required-event rollback, authoritative guard resolution, and lazy identity. + +**See:** [[modules/daemon]], [[commands/daemon]], [[commands/doctor]], +[[state-model]], [[modules/events]], [[modules/markers]], [[testing]] diff --git a/wiki/modules/daemon.md b/wiki/modules/daemon.md index fe8dd9cc..977c5efd 100644 --- a/wiki/modules/daemon.md +++ b/wiki/modules/daemon.md @@ -3,7 +3,7 @@ title: Hive::Daemon type: module source: lib/hive/daemon/ created: 2026-05-06 -updated: 2026-06-20 +updated: 2026-07-24 tags: [daemon, module, automation, dispatcher] --- @@ -23,15 +23,20 @@ the safety-relevant decisions are unit-testable without forking. | `Hive::Daemon::DispatchBaselines` | `lib/hive/daemon/dispatch_baselines.rb` | Crash-safe JSON store for the `[project, slug] → state_file_mtime` baseline map (`daemon_dispatch_baselines.json` under the state home). Atomic write + fail-closed load; mirrors `Hive::UpdateCheck::State`. Stops answered `needs_input` tasks being re-stranded across a daemon restart. | | `Hive::Daemon::StatusConsumer` | `lib/hive/daemon/status_consumer.rb` | Wraps `Open3.capture3("hive status --json")`; returns typed `Row` records including `workflow`. Validates the envelope SHAPE (missing/wrong `schema`, `ok=false`) as a hard `Result(ok: false)`, but tolerates schema-VERSION skew (see "Forward-tolerant schema-version skew" below) so a binary/process version mismatch never crashes a tick. Coerces `tasks[].live_task_lock` to strict boolean so daemon consumers can detect a live runner before a Claude PID is attached, and carries marker attrs so recovery code can preserve `REVIEW_WORKING phase/pass` when rewriting markers. | | `Hive::Daemon::ChildSupervisor` | `lib/hive/daemon/child_supervisor.rb` | Spawns `hive ...` subprocesses with `pgroup: true`; reaps via `Process.wait(-1, WNOHANG)`; parses JSON envelopes from child stdout; supports `terminate_all(grace_sec:)` with TERM→KILL escalation. | -| `Hive::Daemon::Dispatcher` | `lib/hive/daemon/dispatcher.rb` | The poll-classify-dispatch loop. Glues all of the above. Public `tick(now:)` for tests, `run_forever` for production with TERM/INT/HUP signal traps. | +| `Hive::Daemon::Dispatcher` | `lib/hive/daemon/dispatcher.rb` | The poll-classify-dispatch loop. Glues all of the above. Public `tick(now:)` for tests, `run_forever` for production with TERM/INT/HUP signal traps. Guarded automatic-retry requests resolve the authoritative workflow state file from their exact `--stage` and task folder; they never use the ordinary newest-Markdown fallback, so a newer derived `status.md` cannot hide a live `ERROR`. Dispatcher construction also snapshots the lazily computed loaded-source identity for long-running daemon/CLI drift checks. | | `Hive::Daemon::Logger` | `lib/hive/daemon/logger.rb` | One-JSON-line-per-event structured logger. Closed event enum (unknown name raises). Size-rotated. | | `Hive::Daemon::PlanApproval` | `lib/hive/daemon/plan_approval.rb` | Safely turns daemon-enabled `3-plan` approval pauses into `hive develop ... --from 3-plan` dispatches by validating command shape and flipping `WAITING` to `COMPLETE`. | | `Hive::Daemon::StaleAgentHealer` | `lib/hive/daemon/stale_agent_healer.rb` | Rewrites stale `AGENT_WORKING` markers to `ERROR reason=agent_died` or `ERROR reason=agent_orphaned`, while skipping live controller slots and half-migrated projects. It also repairs wedged `REVIEW_WORKING` rows when the recorded Claude child is dead, the review lock holder is still alive, and child-process inspection proves that holder has no remaining children: it logs `reason=review_agent_died` with the original phase/pass, clears the stale marker, terminates the stuck holder, and removes `.lock` so the daemon can retry review normally. Retryable terminal markers such as `8-finalize` `ERROR reason=unpushed_commits` plus non-review terminal agent-loss `ERROR reason=tmux_session_terminated` / `reason=agent_orphaned` are cleared with a bounded per-process retry budget so interrupted sessions can rerun. A narrower timeout path clears `ERROR reason=timeout` exactly once, only on `5-open-pr` and `7-artifacts`, because those re-entries are side-effect-safe (`open_pr_already_open` / idempotent `artifact.md` recollection). `limits_reached` markers (review `REVIEW_ERROR` from reviewers/triage/fix, or single-agent `ERROR` in any stage) self-heal on a cooldown: the writer stamps `retry_after = now + Hive::AgentLimit::RETRY_COOLDOWN_SEC` (default 1h, env `HIVE_LIMITS_RETRY_COOLDOWN_SEC`) and the healer clears them only once `now >= retry_after`, bounded by the same retry budget; cooldown-wait ticks do not burn budget, and a missing/unparseable stamp stays manual. Non-limit operational failures also auto-retry under the same bounded budget so the daemon advances them instead of parking for a human: `ERROR reason=ensure_clean_on_exit_failed` (any worktree-owning stage — the rerun re-applies the scope-checked auto-commit rather than bypassing it, so genuinely out-of-scope residue still re-fails and parks), `REVIEW_ERROR phase=reviewers reason=all_failed` (every reviewer crashed for a non-limit reason; a total usage-limit instead sets `reason=limits_reached` and takes the cooldown path), `REVIEW_ERROR phase=fix reason=fix_failed message="claude stop hook did not signal completion"` for the legacy Claude stop-hook completion bug, and `REVIEW_ERROR phase=fix` auto-commit failures (`fix_auto_commit_scope_failed` / `fix_auto_commit_sign_policy_failed` / `fix_auto_commit_signing_failed`). The integrity/operator reasons `fix_status_check_failed`, `fix_tampered`, generic `fix_failed`, and `dirty_worktree` stay manual. The operator-facing bot/TUI still routes `ensure_clean_on_exit_failed` through `ERROR_MANUAL_ONLY_REASONS` as the post-exhaustion "inspect manually" backstop — the daemon retries first, a human sees it only after the budget is spent. `3-plan` is the special terminal-error case: after any successful terminal `ERROR` clear there, including terminal agent-loss or elapsed `limits_reached`, it queues `hive plan --from 3-plan` through `DispatchRequestQueue` and logs `heal_requeued`, because an empty markerless `plan.md` otherwise classifies straight back to `:error`. | +| `Hive::Daemon::AutoRetry` | `lib/hive/daemon/auto_retry.rb` | Dedicated fail-closed recovery coordinator for exactly two dependency failures: production-shaped execute failure markers correlated with a Codex 401/missing-auth log, and launcher-attributed `claude_launch_failed` markers. It requires an enabled, non-legacy project, stage-specific untouched/clean output, bounded doctor plus reason probes, a durable two-attempt task-event budget, changed health plus 30-minute backoff for attempt two, then queues a marker-guarded durable same-stage request before clearing the exact `marker_id`. Within a tick, project/reason peers reuse an aggregate before another runner is constructed; the 600-second cache is separately scoped per task/reason so a newly parked task cannot inherit another task's stale result. The automatic `marker_cleared` event is required: if it cannot be appended, `Markers.clear_current` restores the original marker and the coordinator reports an error instead of success. A restart resumes an existing pre-clear reservation/request instead of duplicating it. It is separate from `StaleAgentHealer`. | +| `Hive::Daemon::AutoRetryPolicy` | `lib/hive/daemon/auto_retry_policy.rb` | Exact reason classifier and conservative stage-safety predicates. Codex recovery requires both the configured execute profile and the failure marker to identify `agent=codex`; the marker's `log_file` basename selects the exact implementer log for that episode, rather than an mtime-based newest-log guess. Worktree stages require a parseable explicit pointer contained under the configured root, registered by the project repository, and clean under a deadline. Unknown diagnostics, arbitrary plan frontmatter/content, answered brainstorms, legacy unattributed launcher markers, and marker races fail closed. | +| `Hive::Daemon::AutoRetryProbe` | `lib/hive/daemon/auto_retry_probe.rb` | Structured, redacted, output-bounded health chain. All reasons require the in-process required-agent/skill doctor subset, which checks configured profile binaries, minimum versions, preflights, and skill inventory. Codex also requires login plus an ephemeral, read-only, outside-Git-safe smoke in the normal user configuration with model-generated shell inheritance disabled; Claude requires the packaged wrapper, readiness canary, loaded-source-manifest daemon/CLI identity, and runnable Claude binary. Cheap fingerprints include only signals relevant to the current reason, so a Claude-only binary/wrapper change does not invalidate Codex recovery and vice versa. Health fingerprints exclude volatile command output. | +| `Hive::Daemon::AutoRetryHistory` | `lib/hive/daemon/auto_retry_history.rb` | Reconstructs retry episodes from task `events.jsonl`. Reservations precede queue/marker mutation and survive daemon restart; only a successful manual marker clear resets the matching task/reason episode. | +| `Hive::Daemon::BoundedCommand` | `lib/hive/daemon/bounded_command.rb` | Shared external-command capture for auto-retry probes and worktree safety. Continuously drains stdout/stderr pipes while retaining only the first configured bytes per stream, so noisy processes cannot grow temporary storage without bound. It polls the leader against a monotonic deadline, terminates/reaps the full process group, and closes readers after leader exit so descendants cannot hold capture open. | | `Hive::Daemon::DisplayNameBackfiller` | `lib/hive/daemon/display_name_backfiller.rb` | Tick-time self-heal for tasks whose one-shot name generation at `hive new` never landed (agent/codex outage). Re-spawns fire-and-forget `hive generate-name ` for any row whose `Hive::TaskMeta` `display_name` is nil/blank, mirroring `Hive::Commands::New#spawn_name_generator` (detached, pgroup, logged to `/logs/display-name.log`, fully rescued). Anti-churn: an `@inflight` map stores `{pid, at}` per folder, uses `kill(0)` liveness plus `MAX_INFLIGHT_AGE_SEC = 120` to avoid both double-spawns and reused-pid/EPERM pinning, `max_per_tick` (default 2) bounds spawns, and a set name is a natural fixed point. Unexpected row/reap/spawn errors degrade through `:fatal` logging while preserving the no-raise tick contract. Purely additive — never touches markers or dispatch. Logs `display_name_backfill`. | | `Hive::Daemon::TaskIdBackfiller` | `lib/hive/daemon/task_id_backfiller.rb` | Tick-time self-heal for tasks created outside `hive new` (hand-made folder, one `mv`-ed in) whose `meta.yml` has no `id` — `hive new` allocates ids from `Hive::TaskCounter`, so a task that skipped it shows a blank id everywhere (TUI, status, digest, dependency refs). For any row whose `Hive::TaskMeta` `id` is nil it allocates `TaskCounter.next!`, writes it via `TaskMeta.update_id` (every other meta field preserved), and commits the meta on `hive/state` under the per-project commit lock (`Hive::Lock.with_commit_lock`, as every durable committer does) with the per-task `hive_commit(stage_name:, slug:, action: "id-assigned")` call. The `task_id_backfill` event carries `committed:` so a swallowed commit (lock timeout / git error) is visible rather than masquerading as fully durable. Synchronous (no spawn/inflight — assignment is instant), `max_per_tick` (default 5) bounds the per-tick commits, and an assigned id is a natural fixed point. Guards `File.directory?(folder)` first so a row that outlived its folder (e.g. `hive drop` between snapshot and tick) is NOT resurrected by `TaskMeta.write`'s `mkdir_p`. Row/commit errors degrade through `:fatal` / `task_id_backfill_commit_skipped` logging while preserving the no-raise tick contract. Purely additive — never touches markers or dispatch. Logs `task_id_backfill`. | | `Hive::Daemon::PrMergeWatcher` | `lib/hive/daemon/pr_merge_watcher.rb` | Polls `gh pr view --json state` for tasks at 8-finalize/`:complete` and for a narrow set of finalize `ERROR` rows whose PR can still be retired after merge (`git_status_failed`, `claude_launch_failed`). On `MERGED` returns an archive dispatch entry the dispatcher fires. Backs off + drops on persistent gh failures. | | `Hive::Daemon::DigestScheduler` | `lib/hive/daemon/digest_scheduler.rb` | Global daily shipped-digest cadence. Persists `last_digested_date` in `/digest_state.json`, applies a first-run no-history guard, computes owed local calendar days after midnight, caps catch-up with `digest.max_catchup_days`, and emits one `hive digest --date D --json` dispatch at a time. | -| `Hive::Daemon::DispatchRequestQueue` | `lib/hive/daemon/dispatch_request_queue.rb` | File-backed queue (`/dispatch_requests/*.json`) of dispatch requests written by producer paths (Telegram bot via `Hive::Bot::DispatchRequestWriter`, hivebox stage-run dispatches, and the 3-plan healer requeue) and consumed by the dispatcher's tick loop. Current wire schema is `hive-dispatch-request.v2`: `requestor` is the closed enum `bot|healer`, and any other `schema_version` is rejected as `unknown_schema_version`. Allowlists state-mutating verbs (`run develop brainstorm plan review open-pr artifacts finalize archive markers`); rejects everything else with a logged `:dispatch_request_rejected` event. The single-dispatcher invariant lives here: producers write, the daemon dispatches. See [[architecture]] §"Single-dispatcher contract". | +| `Hive::Daemon::DispatchRequestQueue` | `lib/hive/daemon/dispatch_request_queue.rb` | File-backed queue (`/dispatch_requests/*.json`) of dispatch requests written by producer paths and consumed by the dispatcher. Current strict wire schema is `hive-dispatch-request.v3`: `requestor` remains `bot|healer`, and the optional `expected_cleared_marker_id` makes auto-retry requests durable and dispatchable only after the exact classified marker is absent. Older daemons reject v3 rather than ignoring the guard. Allowlists state-mutating verbs (`run develop brainstorm plan review open-pr artifacts finalize archive markers`); rejects everything else with a logged `:dispatch_request_rejected` event. | | `Hive::Daemon::QueueDirectory` | `lib/hive/daemon/queue_directory.rb` | Shared `directory_for(dirname:, state_home:)` helper used by both dispatch queues so the owner-only (0700) per-queue directory invariant — the de-facto auth boundary for the dispatch channel — lives in one place (#253). | | `Hive::Commands::Daemon` | `lib/hive/commands/daemon.rb` | Thor subcommand surface (`start` / `stop` / `status` / `reload` / `tail` / `install` / `enable` / `disable` / `queue`). Owns PID/signal lifecycle, service installation, per-project enrollment, and read-only dispatch-request queue inspection. `queue` delegates to `Hive::Commands::Daemon::QueueCommand`. | | `Hive::Commands::Daemon::QueueCommand` | `lib/hive/commands/daemon/queue_command.rb` | Extracted read-only queue-inspection surface (`hive daemon queue list/show/prune`) — touches only `queue_args`/`json`/`hive_home`, orthogonal to the daemon lifecycle, mirroring the `ServiceInstaller` extraction (#254). Internal IO/parse failures are wrapped in `Hive::InternalError` (exit 70). | @@ -51,11 +56,53 @@ hive daemon start ├─ Hive::Daemon::PrMergeWatcher (Open3.capture3 gh pr view) ├─ Hive::Daemon::DigestScheduler (/digest_state.json) ├─ Hive::Daemon::StaleAgentHealer (AGENT_WORKING repair) + ├─ Hive::Daemon::AutoRetry (fixed dependency-marker recovery) ├─ Hive::Daemon::DisplayNameBackfiller (missing display_name retry) ├─ Hive::Daemon::TaskIdBackfiller (missing meta id assign) └─ Hive::Daemon::Policy (pure decisions) ``` +## Recoverable dependency markers + +`AutoRetry` runs after stale-marker normalization and before pending dispatch +requests. The allowlist is fixed: + +- `implementer_failed` only at `4-execute`, when the current marker has the + production `status=error, message=exit_code=N` shape and the newest execute + implementer log contains a 401 plus the recognized missing bearer/basic-auth + diagnostic. +- `claude_launch_failed` only when the stage launcher boundary stamped + `writer=stages_base_claude_launcher` and `exception_class=Hive::AgentError`. + +Every candidate must pass a stage safety predicate. Brainstorm cannot contain +answered Q&A; plan cannot contain frontmatter or substantive plan/feedback +text; execute and later worktree-owning stages require an explicit, +root-contained, repository-registered, completely clean Git worktree. +Unknown stages, unreadable state, and ambiguous partial output stay parked. + +Healthy attempt one is immediate. Attempt two requires a different full health +fingerprint and at least 1,800 seconds since attempt one. Two reservations +exhaust the task/reason episode across daemon restarts until +`hive markers clear ... --name ERROR --match-attr marker_id=...` succeeds. +Unchanged dependency probes are reused for 600 seconds; relevant config, +binary, environment-selection, wrapper, or required-skill signals trigger an +immediate re-probe. Cache hits preserve the original probe timestamp, so a +stable unhealthy result is refreshed at 600 seconds. Probe errors and timeouts +are unhealthy. + +The global kill switch is: + +```yaml +daemon: + auto_retry: + enabled: false +``` + +It defaults to `true`. Disabled ticks do not classify, probe, audit, enqueue, +or clear. Decisions appear in task `events.jsonl` and daemon JSON logs as +`auto_retry_decision` / `auto_retry_action`; unchanged negative decisions are +throttled to one per marker/fingerprint/action every 600 seconds. + `run_forever` wakes at `daemon.fast_poll_sec` (default 1s) for a cheap probe: non-blocking child reap plus mtime stats of state files and stage directories seen on the last full status scan. A child exit or mtime @@ -401,12 +448,11 @@ validates the argv against an allowlist archive markers`), threads the `request_id` through `spawn → reap` so `reap_completed` can unlink the file and log the lifecycle: -The current strict schema is `hive-dispatch-request.v2`. v2's shape change is -small but breaking by design: `requestor` now accepts `healer` in addition to -`bot`, because the stale-agent healer writes plan rerun requests into the same -queue. The parser is strict-version-matched rather than tolerant here; a -producer that emits a new queue shape requires a coordinated daemon update and -a new schema file before live requests are written. +The current strict schema is `hive-dispatch-request.v3`. v2 added +`requestor=healer`; v3 adds `expected_cleared_marker_id` for dependency +retries. The parser is strict-version-matched rather than tolerant here, so a +stale daemon rejects a guarded request instead of dispatching it without the +new safety rule. ``` :dispatch_request_observed request_id=… project=… slug=… @@ -424,11 +470,15 @@ Lifecycle gates inside `process_dispatch_requests`: 1. Allowlist (`valid_argv?`) — invalid → reject + remove. 2. Expiry (`DispatchRequestQueue::EXPIRY_SEC` = 600s) — old → expire + remove. 3. `find_project` lookup — unknown → reject + remove. -4. `controller.running_task?` — already in flight for this slug → +4. Project enrollment and legacy-layout gates — disabled/half-migrated + projects stay blocked. +5. Optional marker guard — matching live marker blocks; absent marker allows; + replacement/unreadable state rejects. +6. `controller.running_task?` — already in flight for this slug → blocked, file stays for the next tick. -5. `controller.can_dispatch?` gate (caps / cooldown / quarantine) — +7. `controller.can_dispatch?` gate (caps / cooldown / quarantine) — blocked → file stays for the next tick. -6. Otherwise → spawn via `dispatch_command`, threading `request_id` +8. Otherwise → spawn via `dispatch_command`, threading `request_id` into `ChildSupervisor#spawn` and `ChildExit#request_id`. `reap_completed` always refreshes the controller's @@ -447,7 +497,7 @@ restart — re-running work that may already have completed. The fix (`DispatchRequestQueue.claim`) renames the file to `.json.claimed` before the daemon spawns the child. The claimed JSON stays schema-valid for the dispatch-request version the producer wrote -(current writers emit `hive-dispatch-request.v2`); mutable claim metadata +(current writers emit `hive-dispatch-request.v3`); mutable claim metadata (`pid`, `process_start_time`, `claimed_at`) lives in a sibling `.json.claimed.claim` sidecar that is updated after spawn. Claimed files are invisible to `pending` (the glob matches `*.json`, not diff --git a/wiki/modules/events.md b/wiki/modules/events.md index 841b623e..05dbe4fa 100644 --- a/wiki/modules/events.md +++ b/wiki/modules/events.md @@ -3,7 +3,7 @@ title: Hive::Events type: module source: lib/hive/events.rb created: 2026-05-23 -updated: 2026-05-23 +updated: 2026-07-24 tags: [module, events, observability, status, append-only] --- @@ -20,6 +20,9 @@ tags: [module, events, observability, status, append-only] | `error` | `Stages::Base.with_stage_events` rescue path; `emit_marker_event` for error markers | Stage raised, or marker landed on `:error` / `:review_error` / `:review_ci_stale` / `:review_stale` | | `round_waiting` | `Stages::Base.emit_marker_event` | Brainstorm or plan stage closed with `:waiting` marker | | `round_complete` | same | Brainstorm or plan stage closed with `:complete` marker | +| `auto_retry_decision` | `Hive::Daemon::AutoRetry` | Positive or throttled negative dependency-recovery decision | +| `auto_retry_reserved` | `Hive::Daemon::AutoRetryHistory` | Durable, pre-mutation automatic-attempt reservation | +| `marker_cleared` | `hive markers clear` or `Hive::Daemon::AutoRetry` | Successful guarded clear, attributed as `manual` or `daemon_auto_retry` | `ROUND_EVENT_STAGES = %w[brainstorm plan]` is the registry that gates round events — adding a new stage that publishes `:waiting` / `:complete` round markers requires extending this list so `emit_marker_event` stays in sync with the producers. @@ -34,12 +37,28 @@ tags: [module, events, observability, status, append-only] - `agent` — `" "` for agent spawns (e.g. `"claude review-stub-reviewer-pass01"`); `"phase= pass="` for review phase brackets; `null` for stage / round / error events. - `event_type` — one of the table above. - `message` — short human-readable detail. `agent_start` carries `cwd= timeout_sec=N max_budget_usd=N` (full path, not basename — basename collapsed the worktree-vs-task-folder distinction); `agent_end` carries `status=… exit_code=… pid=…` (or `status=exception` with the error class); `stage_exit` carries `status= phase=… reason=… pass=…` when those marker attrs are present. +- `details` — optional bounded JSON object (maximum 8 KiB). Legacy callers that + omit it retain the original byte shape. Auto-retry records use it for task id/slug, + stage, marker identity/reason, actor, redacted probe summaries, full health + fingerprint, attempt, action, rationale, request id, and decision timestamp. + +For retry history, `auto_retry_reserved` is the durable budget entry. +`marker_cleared details.actor=manual` resets only the matching marker reason; +an automatic clear is audited but does not reset its own episode. ## Storage and atomicity - **Path**: `/events.jsonl` (one file per task slug, lives next to `task.md`). - **Append**: single `File.write` of `JSON.generate(record) + "\n"` opened `O_WRONLY | O_APPEND | O_CREAT`. Records stay well under `PIPE_BUF` (~4 KiB), so POSIX append-atomicity holds across concurrent emitters; the single-write contract is load-bearing and must not be split into "write JSON then write newline." -- **Failure mode**: `SystemCallError` during emit is caught and warned to stderr (`[hive.events] failed to emit ...`). The producing stage / agent control flow is not interrupted — observability must never mask the underlying run result. +- **Failure mode**: ordinary `emit` catches filesystem/IO failures and warns so + observability does not mask a stage result. `emit_required` is the narrow + durable-transition variant used by automatic retry reservations, automatic + marker clears, and manual retry-history resets: append failures propagate, + while a later derived + `status.md` render failure only warns because `events.jsonl` is authoritative. + Automatic retry performs its clear through the transactional block form of + `Markers.clear_current`; if this required append fails, the original marker + body is restored. ## Derived `status.md` diff --git a/wiki/modules/markers.md b/wiki/modules/markers.md index fc8e0a32..267c515e 100644 --- a/wiki/modules/markers.md +++ b/wiki/modules/markers.md @@ -3,7 +3,7 @@ title: Hive::Markers type: module source: lib/hive/markers.rb created: 2026-04-25 -updated: 2026-06-18 +updated: 2026-07-24 tags: [marker, protocol, flock] --- @@ -35,6 +35,20 @@ Allowlist: see `KNOWN_NAMES` in `lib/hive/markers.rb`. `ERROR` markers written through `Markers.set` receive a generated `marker_id` attr unless the caller supplies one. This is the high-cardinality recovery discriminator for `hive markers clear --match-attr marker_id=...`; legacy rows without it fall back to observed attrs such as `reason=exit_code,exit_code=143`. +The daemon's dedicated dependency auto-retry path accepts only current +nonblank-`marker_id` `ERROR` markers whose exact diagnostics prove either a +Codex authentication failure behind `implementer_failed` or a launcher- +attributed Claude failure behind `claude_launch_failed`. It never clears by +reason alone. Execute failures also carry `agent=` and +`log_file=`; Codex recovery requires both configured +and recorded agents to be Codex and reads only that episode's named log. +After health and work-safety gates pass, it enqueues the normal +same-stage run and calls `clear_current(..., match_attrs: +{"marker_id" => current_id})`; a mismatch leaves the replacement marker intact +and withdraws the request only if it is still unclaimed. The automatic clear +uses `clear_current`'s block form so failure to append its required task event +restores the original state-file body. + `KILL_CLASS_EXIT_CODES = %w[130 137 143]` — POSIX signal exit codes (SIGINT/SIGKILL/SIGTERM). Only an `ERROR` marker shaped as `reason=exit_code exit_code=130|137|143` means the task was interrupted rather than broken. Same numeric codes with another reason remain structured recoverable failures. The numeric list is shared by `Hive::Tui::BubbleModel#auto_heal_kill_class_errors` (auto-clears explicit signal-kill markers) and `Hive::Tui::KeyMap.error_message` (routes Enter to OpenLogTail instead of RecoverError so Enter doesn't race the auto-healer for the markers-lock). Regex: `MARKER_RE` enumerates every name in `KNOWN_NAMES`, requires a marker-name boundary, and captures attrs until the terminating `-->` without crossing another `