diff --git a/config.example.yml b/config.example.yml index 808c665a..026bd918 100644 --- a/config.example.yml +++ b/config.example.yml @@ -7,6 +7,14 @@ registered_projects: [] screenote: base_url: https://screenote.ai +# Global, default-on kill switch for the conservative health-gated daemon +# recovery coordinator. Set false to park existing and future automatic +# clear/rerun requests without changing per-project daemon enrollment or the +# older stale-agent healer. +daemon: + auto_retry: + enabled: true + # Project .hive-state/config.yml files can opt Claude-backed stages into # permission presets with `permissions:`. See docs/permissions.md for the # yolo/read-only/scoped reference and the tool-level caveat. diff --git a/lib/hive/bot/handlers/recovery_sequence.rb b/lib/hive/bot/handlers/recovery_sequence.rb index b83885cd..b2de9a9d 100644 --- a/lib/hive/bot/handlers/recovery_sequence.rb +++ b/lib/hive/bot/handlers/recovery_sequence.rb @@ -180,24 +180,11 @@ module Hive # would exit 4. Both branches intentionally diverge from the # pre-U7 clear_and_retry path. def self.retry_commands(project:, slug:, stage:, marker:, match_attr: nil, workflow: nil) - verb = retry_verb_for_stage(stage, workflow: workflow, project: project) - return [] unless verb - - commands = [] - marker_name = marker.to_s - unless marker_name.casecmp("none").zero? || marker_name.casecmp("agent_working").zero? - clear_argv = [ "hive", "markers", "clear", slug, "--name", marker_name.upcase, - "--project", project ] - clear_argv += [ "--match-attr", match_attr ] if match_attr.to_s.include?("=") - clear_argv << "--json" - commands << clear_argv - end - # `hive run` (the generic stage runner) scopes by --stage and has no - # --from; the coding advance/recovery verbs assert the source stage - # with --from. - stage_flag = verb == "run" ? "--stage" : "--from" - commands << [ "hive", verb, slug, stage_flag, stage, "--project", project, "--json" ] - commands + require "hive/recovery_sequence" + Hive::RecoverySequence.commands( + project: project, slug: slug, stage: stage, marker: marker, + match_attr: match_attr, workflow: workflow + ) end def self.alert_reset(project, slug, stage, marker = nil, match_attr = nil) diff --git a/lib/hive/claude_launcher.rb b/lib/hive/claude_launcher.rb index 15b092ae..6435af7e 100644 --- a/lib/hive/claude_launcher.rb +++ b/lib/hive/claude_launcher.rb @@ -482,13 +482,33 @@ module Hive ENV.fetch("HIVE_TMUX_BIN", "tmux") end + def interactive_wrapper_path + File.expand_path("scripts/interactive_claude_wrapper.sh", __dir__) + end + + # Uses the production readiness detector over shipped, deterministic + # samples. Health checks call this API rather than maintaining a second, + # weaker String#include? detector that can drift from launch behavior. + def readiness_detector_self_check + positive = "Claude Code\n❯ \n" + negative = "Claude Code\nQuick safety check\nYes, I trust this folder\n❯ \n" + positive_ok = claude_ready_prompt?(positive) + negative_ok = !claude_ready_prompt?(negative) + { + ok: positive_ok && negative_ok, + detail: positive_ok && negative_ok ? "production_samples_ok" : "production_samples_failed" + } + rescue StandardError => e + { ok: false, detail: e.class.name } + end + def wrapper_command(cwd:, add_dirs:, profile:, permission_mode:, allowed_tools: DEFAULT_ALLOWED_TOOLS, disallowed_tools: nil, cli_flags: [], mcp_config_path: nil, strict_mcp_config: false) command = [ "bash", - File.expand_path("scripts/interactive_claude_wrapper.sh", __dir__), + interactive_wrapper_path, "--cwd", cwd ] Array(add_dirs).each { |dir| command.concat([ "--add-dir", dir ]) } diff --git a/lib/hive/cli.rb b/lib/hive/cli.rb index cd809068..74e62237 100644 --- a/lib/hive/cli.rb +++ b/lib/hive/cli.rb @@ -46,7 +46,12 @@ module Hive desc "version", "Print hive version" def version - puts Hive::VERSION + if options[:json] + require "hive/runtime_identity" + puts JSON.generate(Hive::RuntimeIdentity.snapshot) + else + puts Hive::VERSION + end end map "--version" => :version @@ -977,16 +982,19 @@ module Hive option :project, type: :string, desc: "scope slug lookup to one registered project" option :match_attr, type: :string, desc: "refuse clear unless marker has attr(s): KEY=VALUE[,KEY=VALUE]" + option :recovery_id, type: :string, + desc: "internal auto-retry recovery id (daemon only; skips manual rearm)" def markers(subcommand, target = nil) require "hive/commands/markers" - Hive::Commands::Markers.new( - subcommand, - target, + kwargs = { name: options[:name], project: options[:project], match_attr: options[:match_attr], json: options[:json] - ).call + } + recovery_id = options[:recovery_id] + kwargs[:recovery_id] = recovery_id unless recovery_id.nil? + Hive::Commands::Markers.new(subcommand, target, **kwargs).call end desc "babysit SUBCOMMAND [PROJECT]", "Manage the experimental PR babysitter" diff --git a/lib/hive/commands/doctor.rb b/lib/hive/commands/doctor.rb index 99afbc08..dc2203be 100644 --- a/lib/hive/commands/doctor.rb +++ b/lib/hive/commands/doctor.rb @@ -45,6 +45,16 @@ module Hive # JSON encoder. Returns `nil` before `#call` has populated it. attr_reader :rows + # Lightweight in-process API shared by the CLI and daemon health gate. + # It deliberately omits optional qmd/migration warnings and accepts + # already-bounded dependency rows so daemon callers never shell out + # through Doctor's human-oriented probes. + def self.required_agent_rows(config:, project_root:, dependency_rows: nil) + new(config: config, project_root: project_root).required_agent_rows( + dependency_rows: dependency_rows + ) + end + def initialize(config:, project_root:, json: false, output: $stdout) @config = config @project_root = project_root @@ -71,6 +81,11 @@ module Hive EXIT_CONFIG_ERROR end + def required_agent_rows(dependency_rows: nil) + dependencies = dependency_rows.nil? ? check_tmux : Array(dependency_rows) + dependencies + check_stages + check_reviewers + end + private def failing_status?(status) diff --git a/lib/hive/commands/markers.rb b/lib/hive/commands/markers.rb index f5df7df1..50928476 100644 --- a/lib/hive/commands/markers.rb +++ b/lib/hive/commands/markers.rb @@ -42,13 +42,22 @@ module Hive VALID_SUBCOMMANDS = %w[clear].freeze - def initialize(subcommand, target = nil, name: nil, project: nil, match_attr: nil, json: false) + # Lightweight row shape for reusing AutoRetry::Safety without depending + # on StatusConsumer during an in-process clear. + SafetyRow = Struct.new(:project, :slug, :stage, :folder, :state_file, :marker, + :marker_attrs, :live_task_lock, :worktree_path, :id, + :task_lock_owned, + keyword_init: true) + + def initialize(subcommand, target = nil, name: nil, project: nil, match_attr: nil, json: false, + recovery_id: nil) @subcommand = subcommand @target = target @name = name @project_filter = project @match_attr = match_attr @json = json + @recovery_id = recovery_id end def call @@ -104,19 +113,31 @@ module Hive # erases that fresh marker. The hive_commit follows under # `with_commit_lock` to serialise hive/state branch writes # against any concurrent committer (auto-heal, run loop). - Hive::Markers.with_markers_lock(task.state_file) do - marker = Hive::Markers.current(task.state_file) - actual = marker.name.to_s.upcase - unless actual == normalized - raise Hive::WrongStage, - "hive markers clear: task #{task.slug} has marker #{actual.inspect}, " \ - "not #{normalized.inspect}; refusing to clear (the file may have been edited)." + cleared_attrs = nil + with_auto_retry_task_lock(task) do |task_lock_owned| + Hive::Markers.with_markers_lock(task.state_file) do + marker = Hive::Markers.current(task.state_file) + actual = marker.name.to_s.upcase + unless actual == normalized + raise Hive::WrongStage, + "hive markers clear: task #{task.slug} has marker #{actual.inspect}, " \ + "not #{normalized.inspect}; refusing to clear (the file may have been edited)." + end + + match_attr_or_raise!(task, marker) + auto_retry_safety_or_raise!(task, marker, task_lock_owned: task_lock_owned) if auto_retry_clear? + cleared_attrs = marker.attrs.dup + # Durable retry accounting is a write-ahead transition. If it + # cannot be persisted, the marker remains intact and the command + # fails closed. Reconciliation tolerates the opposite crash window + # (ledger advanced but marker removal not yet committed). + persist_auto_retry_ledger!(task, cleared_attrs) + Hive::Markers.remove_marker(task.state_file, marker.raw) end - - match_attr_or_raise!(task, marker) - Hive::Markers.remove_marker(task.state_file, marker.raw) end + emit_manual_clear_audit(task, cleared_attrs) unless auto_retry_clear? + Hive::Lock.with_commit_lock(task.hive_state_path) do record_hive_commit(task, normalized) end @@ -124,6 +145,122 @@ module Hive emit_success(task, normalized) end + def auto_retry_clear? + !@recovery_id.nil? && !@recovery_id.to_s.strip.empty? + end + + # Automatic clear: re-run stage safety under current on-disk state and + # require the durable intent to carry this recovery_id. Never rearms. + def auto_retry_safety_or_raise!(task, marker, task_lock_owned:) + require "hive/daemon/auto_retry/state" + require "hive/daemon/auto_retry/safety" + + reason = marker.attrs["reason"].to_s + marker_id = marker.attrs["marker_id"].to_s + unless Hive::Config.load_global_daemon.dig("auto_retry", "enabled") != false + raise Hive::WrongStage, + "hive markers clear: daemon.auto_retry.enabled is false; refusing automatic clear" + end + state = Hive::Daemon::AutoRetry::State.new + task_key = Hive::Daemon::AutoRetry::State.task_key(task_id: task.id, slug: task.slug) + record = state.get( + project: project_name_for(task), + task_key: task_key, + stage: "#{task.stage_index}-#{task.stage_name}", + reason: reason + ) + unless record.is_a?(Hash) && record["recovery_id"].to_s == @recovery_id.to_s && + %w[clear_queued marker_cleared].include?(record["phase"].to_s) + raise Hive::WrongStage, + "hive markers clear: recovery_id #{@recovery_id.inspect} does not match " \ + "a durable auto-retry intent; refusing automatic clear" + end + + row = SafetyRow.new( + project: project_name_for(task), + slug: task.slug, + stage: "#{task.stage_index}-#{task.stage_name}", + folder: task.folder, + state_file: task.state_file, + marker: marker.name.to_s, + marker_attrs: marker.attrs, + live_task_lock: task_lock_owned, + task_lock_owned: task_lock_owned, + worktree_path: (task.worktree_path rescue nil), + id: task.id + ) + gate = Hive::Daemon::AutoRetry::Safety.new.check( + row: row, + observed_marker_id: marker_id, + observed_reason: reason, + re_read: true + ) + return if gate.safe? + + raise Hive::WrongStage, + "hive markers clear: auto-retry safety failed (#{gate.rationale}); " \ + "refusing to clear" + end + + def persist_auto_retry_ledger!(task, cleared_attrs) + require "hive/daemon/auto_retry/state" + reason = (cleared_attrs || {})["reason"].to_s + return if reason.empty? + + state = Hive::Daemon::AutoRetry::State.new + task_key = Hive::Daemon::AutoRetry::State.task_key(task_id: task.id, slug: task.slug) + project = project_name_for(task) + stage = "#{task.stage_index}-#{task.stage_name}" + + if auto_retry_clear? + # Advance phase without resetting attempt budget. + state.upsert!( + project: project, task_key: task_key, stage: stage, reason: reason, + phase: "marker_cleared" + ) + else + # Successful manual clear rearms the two-attempt budget. + state.rearm_manual!( + project: project, task_key: task_key, stage: stage, reason: reason + ) + end + end + + def with_auto_retry_task_lock(task) + return yield(false) unless auto_retry_clear? + + Hive::Lock.with_task_lock( + task.folder, + "op" => "auto_retry_clear", + "recovery_id" => @recovery_id.to_s + ) { yield(true) } + end + + def emit_manual_clear_audit(task, cleared_attrs) + require "hive/events" + reason = (cleared_attrs || {})["reason"].to_s + return if reason.empty? + + Hive::Events.emit( + task_folder: task.folder, + slug: task.slug, + stage: "#{task.stage_index}-#{task.stage_name}", + event_type: :auto_retry_decision, + agent: "operator", + message: "manual_rearm:manual_clear", + details: { + action: "manual_rearm", rationale: "manual_clear", + marker_reason: reason, marker_id: (cleared_attrs || {})["marker_id"], + attempts_rearmed: 2 + } + ) + end + + def project_name_for(task) + match = Hive::Config.registered_projects.find { |p| p["path"] == task.project_root } + match ? match["name"] : File.basename(task.project_root) + end + # Cross-process race guard. The TUI observes an ERROR marker at # time T and dispatches `hive markers clear` at T+1s. If a # concurrent `hive run` writes a fresh ERROR marker in that diff --git a/lib/hive/config.rb b/lib/hive/config.rb index c686876b..27fdddae 100644 --- a/lib/hive/config.rb +++ b/lib/hive/config.rb @@ -313,6 +313,11 @@ module Hive "daemon" => { "enabled" => false, "autostart" => false, + # Global kill switch for the v1 health-gated auto-retry coordinator. + # Defaults on; does not change per-project daemon.enabled or healer. + "auto_retry" => { + "enabled" => true + }, "poll_interval_sec" => 30, "fast_poll_sec" => 1, "edit_debounce_sec" => 30, @@ -2237,6 +2242,25 @@ module Hive end validate_daemon_verb_timeouts!(daemon, source_path) + validate_daemon_auto_retry!(daemon, source_path) + end + + def validate_daemon_auto_retry!(daemon, source_path) + auto_retry = daemon["auto_retry"] + return if auto_retry.nil? + + unless auto_retry.is_a?(Hash) + raise ConfigError, + "daemon.auto_retry in #{describe_source(source_path)} must be a Hash; " \ + "got #{auto_retry.inspect} (#{auto_retry.class})" + end + + enabled = auto_retry["enabled"] + return if enabled.nil? || enabled == true || enabled == false + + raise ConfigError, + "daemon.auto_retry.enabled in #{describe_source(source_path)} must be a boolean " \ + "(true / false); got #{enabled.inspect} (#{enabled.class})" end def validate_web_config!(cfg, source_path) diff --git a/lib/hive/daemon/auto_retry/classifier.rb b/lib/hive/daemon/auto_retry/classifier.rb new file mode 100644 index 00000000..8b308405 --- /dev/null +++ b/lib/hive/daemon/auto_retry/classifier.rb @@ -0,0 +1,356 @@ +# frozen_string_literal: true + +require "hive/task_action" +require "hive/stages/base" +require "hive/config" + +module Hive + module Daemon + module AutoRetry + # Pure fail-closed classifier for v1 health-gated auto-retry. + # Maps a terminal ERROR row into one of: + # - :codex_auth_401 + # - :claude_launcher + # - :ineligible + # without launching probes or reading the network. Diagnostic text is + # evidence only; it never becomes command construction input. + class Classifier + RECOVERY_CODEX_AUTH = :codex_auth_401 + RECOVERY_CLAUDE_LAUNCHER = :claude_launcher + RECOVERY_INELIGIBLE = :ineligible + + # Exact Codex 401 authentication signature. Require HTTP 401 plus + # either missing-bearer or basic-auth wording so an unrelated 401 + # (or a 401 without auth-scheme language) stays parked. + CODEX_AUTH_HTTP = /\b(?:http(?:\/\d(?:\.\d)?)?[\s:-]*|status(?:\s+code)?[\s:=]*)401\b/i + CODEX_AUTH_BEARER = /missing\s+(?:a\s+)?bearer(?:[-\s]?auth(?:entication|orization)?)?| + no\s+bearer(?:[-\s]?token)?| + bearer\s+(?:token\s+)?(?:missing|required|not\s+found)| + authorization\s+header\s+is\s+missing/ix + CODEX_AUTH_BASIC = /basic[-\s]?auth(?:entication|orization)?| + missing\s+(?:a\s+)?basic(?:[-\s]?auth)?| + www-authenticate:\s*basic/ix + + # Claude launcher/packaging/readiness signatures accepted for v1. + # Session-name collisions, quota walls, and permission prompts are + # excluded so those failures remain manual. + CLAUDE_LAUNCHER_SIGNATURES = [ + /claude interactive prompt did not become ready/i, + /could not inspect claude tmux session/i, + /tmux session .+ did not start/i, + /wrapper\s+(?:script\s+)?(?:missing|not\s+found|not\s+executable|failed)/i, + /readiness\s+(?:detector|check|probe)\s+(?:failed|error|mismatch)/i, + /packag(?:e|ing).*(?:wrapper|script|install)|(?:wrapper|script|install).*packag(?:e|ing)/i + ].freeze + + CLAUDE_EXCLUSIONS = [ + /already exists/i, + /limits?\s+reached|usage[\/\s-]?credit|quota|rate[-\s]?limit|stop and wait for limit/i, + /permission(?:s)?\s+(?:prompt|denied|required)|dangerously-skip-permissions/i + ].freeze + + Result = Struct.new( + :recovery_class, + :rationale, + :marker_reason, + :marker_id, + :stage, + :evidence_kind, + keyword_init: true + ) do + def eligible? + recovery_class != RECOVERY_INELIGIBLE + end + end + + # @param row [StatusConsumer::Row, Hash, #to_h] status row + # @param config [Hash, nil] effective project config + # @param evidence [Hash, nil] machine-oriented evidence: + # :text / "text", :kind / "kind", :fresh (bool), + # :marker_signature_match (bool), :error (symbol/string) + # @param stage_agent [String, Symbol, nil] override for stage agent + # @param execute_agent [String, Symbol, nil] override for execute agent + # @param claude_mode [String, Symbol, nil] override for claude.mode + def self.classify(row:, config: nil, evidence: nil, + stage_agent: nil, execute_agent: nil, claude_mode: nil) + new( + row: row, + config: config, + evidence: evidence, + stage_agent: stage_agent, + execute_agent: execute_agent, + claude_mode: claude_mode + ).classify + end + + def initialize(row:, config: nil, evidence: nil, + stage_agent: nil, execute_agent: nil, claude_mode: nil) + @row = normalize_row(row) + @config = config.is_a?(Hash) ? config : {} + @evidence = normalize_evidence(evidence) + @stage_agent_override = stage_agent + @execute_agent_override = execute_agent + @claude_mode_override = claude_mode + end + + def classify + marker_name = marker_name_of + reason = marker_reason_of + stage = stage_of + + unless marker_name == "error" + return ineligible(:not_error_marker) + end + + case reason + when "implementer_failed" + classify_codex_auth(stage) + when "claude_launch_failed" + classify_claude_launcher(stage) + when nil, "" + ineligible(:missing_reason) + else + ineligible(:unknown_reason) + end + rescue StandardError + # Fail closed: a classifier bug must never make a marker eligible. + ineligible(:classifier_error) + end + + private + + def classify_codex_auth(stage) + unless stage == "4-execute" # coding-scoped: v1 Codex recovery is execute-only + return ineligible(:wrong_stage) + end + + agent = effective_execute_agent + unless agent == "codex" + return ineligible(:wrong_provider) + end + + text = evidence_text + if text.nil? + return ineligible(evidence_rejection_rationale) + end + + unless codex_auth_signature?(text) + return ineligible(:unknown_signature) + end + + eligible(RECOVERY_CODEX_AUTH, :codex_auth_401) + end + + def classify_claude_launcher(stage) + agent = effective_stage_agent + unless agent == "claude" + return ineligible(:wrong_provider) + end + + mode = effective_claude_mode + unless mode.to_s == "tmux" + return ineligible(:wrong_claude_mode) + end + + # v1 only defines replay safety for execute + brainstorm/plan. + unless %w[2-brainstorm 3-plan 4-execute].include?(stage) # coding-scoped: v1 replay proofs + return ineligible(:stage_not_supported) + end + + text = evidence_text + if text.nil? + return ineligible(evidence_rejection_rationale) + end + + if claude_excluded?(text) + return ineligible(:excluded_signature) + end + + unless claude_launcher_signature?(text) + return ineligible(:unknown_signature) + end + + eligible(RECOVERY_CLAUDE_LAUNCHER, :claude_launcher) + end + + def codex_auth_signature?(text) + return false unless text.match?(CODEX_AUTH_HTTP) + + text.match?(CODEX_AUTH_BEARER) || text.match?(CODEX_AUTH_BASIC) + end + + def claude_launcher_signature?(text) + CLAUDE_LAUNCHER_SIGNATURES.any? { |re| text.match?(re) } + end + + def claude_excluded?(text) + CLAUDE_EXCLUSIONS.any? { |re| text.match?(re) } + end + + def evidence_text + return nil if @evidence[:error] + return nil if @evidence.key?(:fresh) && @evidence[:fresh] == false + return nil if @evidence.key?(:marker_signature_match) && + @evidence[:marker_signature_match] == false + + text = @evidence[:text].to_s + marker_msg = marker_message + parts = [ marker_msg, text ].map { |p| p.to_s.strip }.reject(&:empty?) + return nil if parts.empty? + + parts.join("\n") + end + + def evidence_rejection_rationale + case @evidence[:error].to_s + when "stale_diagnostic" then return :stale_diagnostic + when "missing_diagnostic", "missing_folder" then return :missing_diagnostic + when "unreadable_diagnostic" then return :unreadable_diagnostic + when "" then # fall through + else + return :evidence_error if @evidence[:error] + end + return :stale_diagnostic if @evidence.key?(:fresh) && @evidence[:fresh] == false + return :stale_diagnostic if @evidence.key?(:marker_signature_match) && + @evidence[:marker_signature_match] == false + return :missing_diagnostic if @evidence[:text].to_s.strip.empty? && + marker_message.strip.empty? + + :missing_diagnostic + end + + def marker_message + attrs = marker_attrs + attrs["message"].to_s + end + + def effective_execute_agent + return @execute_agent_override.to_s.downcase if @execute_agent_override + + begin + Hive::Stages::Base.stage_profile(@config, "execute").name.to_s.downcase + rescue StandardError + (@config.dig("execute", "agent") || "claude").to_s.downcase + end + end + + def effective_stage_agent + return @stage_agent_override.to_s.downcase if @stage_agent_override + + stage_name = stage_role_name + begin + Hive::Stages::Base.stage_profile(@config, stage_name).name.to_s.downcase + rescue StandardError + (@config.dig(stage_name, "agent") || "claude").to_s.downcase + end + end + + def effective_claude_mode + return @claude_mode_override.to_s if @claude_mode_override + + begin + Hive::Config.claude_mode(@config).to_s + rescue StandardError + (@config.dig("claude", "mode") || "tmux").to_s + end + end + + def stage_role_name + # coding-scoped (block): maps coding descriptor dirs to config role names + case stage_of + when "2-brainstorm" then "brainstorm" + when "3-plan" then "plan" + when "4-execute" then "execute" + when "5-open-pr" then "open_pr" + when "7-artifacts" then "artifacts" + when "8-finalize" then "finalize" + else + stage_of.to_s.sub(/\A\d+-/, "").tr("-", "_") + end + end + + def marker_name_of + raw = @row[:marker] + raw = raw.name if raw.respond_to?(:name) && !raw.is_a?(String) + raw.to_s.downcase + end + + def marker_reason_of + attrs = marker_attrs + attrs["reason"].to_s + end + + def marker_id_of + marker_attrs["marker_id"].to_s + end + + def marker_attrs + attrs = @row[:marker_attrs] || @row[:attrs] || {} + return {} unless attrs.is_a?(Hash) + + attrs.each_with_object({}) do |(k, v), acc| + acc[k.to_s] = v + end + end + + def stage_of + @row[:stage].to_s + end + + def normalize_row(row) + return row if row.is_a?(Hash) && (row.key?(:marker) || row.key?("marker")) + + if row.respond_to?(:to_h) + h = row.to_h + return h.transform_keys(&:to_sym) if h.is_a?(Hash) + end + + { + marker: row.respond_to?(:marker) ? row.marker : nil, + marker_attrs: row.respond_to?(:marker_attrs) ? row.marker_attrs : {}, + stage: row.respond_to?(:stage) ? row.stage : nil, + folder: row.respond_to?(:folder) ? row.folder : nil, + project: row.respond_to?(:project) ? row.project : nil, + slug: row.respond_to?(:slug) ? row.slug : nil + } + rescue StandardError + {} + end + + def normalize_evidence(evidence) + return {} if evidence.nil? + return evidence.transform_keys(&:to_sym) if evidence.is_a?(Hash) + + if evidence.respond_to?(:to_h) + h = evidence.to_h + return h.transform_keys(&:to_sym) if h.is_a?(Hash) + end + + { text: evidence.to_s, kind: :raw } + end + + def eligible(recovery_class, rationale) + Result.new( + recovery_class: recovery_class, + rationale: rationale.to_s, + marker_reason: marker_reason_of, + marker_id: marker_id_of, + stage: stage_of, + evidence_kind: @evidence[:kind] + ) + end + + def ineligible(rationale) + Result.new( + recovery_class: RECOVERY_INELIGIBLE, + rationale: rationale.to_s, + marker_reason: marker_reason_of, + marker_id: marker_id_of, + stage: stage_of, + evidence_kind: @evidence[:kind] + ) + end + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/coordinator.rb b/lib/hive/daemon/auto_retry/coordinator.rb new file mode 100644 index 00000000..f4bf1df7 --- /dev/null +++ b/lib/hive/daemon/auto_retry/coordinator.rb @@ -0,0 +1,755 @@ +# frozen_string_literal: true + +require "digest" +require "securerandom" +require "hive/daemon/auto_retry/classifier" +require "hive/daemon/auto_retry/evidence" +require "hive/daemon/auto_retry/health" +require "hive/daemon/auto_retry/policy" +require "hive/daemon/auto_retry/safety" +require "hive/daemon/auto_retry/state" +require "hive/recovery_sequence" +require "hive/events" +require "hive/markers" + +module Hive + module Daemon + module AutoRetry + # Orchestrates v1 health-gated auto-retry: classify → safety → policy + # → probe → re-check safety → queue guarded clear + same-stage rerun. + class Coordinator + def initialize(logger:, controller:, state: nil, health: nil, + request_queue: nil, enabled: true, + config_for_project: nil, project_enabled: nil, + legacy_projects: nil, clock: -> { Time.now }, + dry_run: false) + @logger = logger + @controller = controller + @state = state || State.new(logger: logger, clock: clock) + @health = health || Health.new(clock: clock) + @request_queue = request_queue || Hive::Daemon::DispatchRequestQueue + @enabled = enabled + @config_for_project = config_for_project || ->(_name) { {} } + @project_enabled = project_enabled || ->(_name) { true } + @legacy_projects = legacy_projects || {} + @clock = clock + @dry_run = dry_run + @probes_this_tick = 0 + @probe_cursor = 0 + end + + attr_accessor :enabled + + def reconfigure!(enabled:) + @enabled = enabled + end + + # Called by the dispatcher immediately before spawning an auto-retry + # same-stage rerun. Persists attempt accounting; returns false if the + # durable write fails (caller must not spawn). + def note_retry_dispatched!(recovery_id:, project:, slug:, stage:, reason:, + fingerprint: nil, task_id: nil) + task_key = State.task_key(task_id: task_id, slug: slug) + rec = @state.get(project: project, task_key: task_key, stage: stage, reason: reason) + return false unless rec.is_a?(Hash) + return false unless rec["recovery_id"].to_s == recovery_id.to_s + fp = fingerprint || rec["pending_fingerprint"] || rec["last_probe_fingerprint"] + @state.mark_dispatched!( + project: project, task_key: task_key, stage: stage, reason: reason, + fingerprint: fp, recovery_id: recovery_id + ) + true + rescue State::Error => e + @logger&.event(:auto_retry_state_error, message: e.message) + false + end + + def tick(rows, now: @clock.call) + @probes_this_tick = 0 + @health.reset_tick_cache! + return unless @enabled + return if @state.suspended? + + rows = Array(rows) + reconcile_records_without_current_stage(rows, now: now) + ordered_rows = fair_rows(rows) + ordered_rows.each do |row| + evaluate_row(row, now: now) + rescue StandardError => e + @logger&.event(:auto_retry_error, + project: row_get(row, :project), + slug: row_get(row, :slug), + error_class: e.class.name, + message: e.message) + end + advance_probe_cursor(rows.length) + end + + # Called after a successful manual markers clear (no recovery_id). + def on_manual_clear!(project:, task_key:, stage:, reason:) + return if reason.to_s.empty? + + @state.rearm_manual!(project: project, task_key: task_key, + stage: stage, reason: reason) + audit(project: project, slug: task_key, stage: stage, + action: "manual_rearm", rationale: "manual_clear", + marker_reason: reason, recovery_class: nil) + rescue State::Suspended => e + @logger&.event(:auto_retry_state_error, message: e.message) + end + + private + + def evaluate_row(row, now:) + project = row_get(row, :project).to_s + slug = row_get(row, :slug).to_s + stage = row_get(row, :stage).to_s + task_key = State.task_key(task_id: row_get(row, :id), slug: slug) + inflight = inflight_record_for(project: project, task_key: task_key, stage: stage) + if inflight + reconcile_inflight(row, inflight, task_key, now: now) + return + end + + marker = row_get(row, :marker).to_s.downcase + return unless marker == "error" + + attrs = row_get(row, :marker_attrs) || {} + reason = (attrs["reason"] || attrs[:reason]).to_s + return if reason.empty? + + folder = row_get(row, :folder).to_s + marker_id = (attrs["marker_id"] || attrs[:marker_id]).to_s + + # Merge-watcher precedence for finalize. + if stage == "8-finalize" # coding-scoped: finalize recovery belongs to merge watcher + return audit_negative(row, "merge_watcher_owned", nil, now: now) + end + + unless @project_enabled.call(project) + return audit_negative(row, "project_disabled", nil, now: now) + end + + config = @config_for_project.call(project) || {} + evidence = Evidence.load( + folder: folder, + state_file: row_get(row, :state_file), + marker_attrs: attrs + ) + classification = Classifier.classify( + row: row, config: config, evidence: evidence.to_h + ) + unless classification.eligible? + return audit_negative(row, classification.rationale, + classification.recovery_class, now: now) + end + + record = @state.get(project: project, task_key: task_key, + stage: stage, reason: reason) || {} + + safety = Safety.new( + controller: @controller, + enabled_projects: nil, + legacy_projects: @legacy_projects + ) + unless @project_enabled.call(project) + return audit_negative(row, "project_disabled", classification.recovery_class, now: now) + end + if @legacy_projects[project] + return audit_negative(row, "legacy_layout", classification.recovery_class, now: now) + end + + gate = safety.check( + row: row, + observed_marker_id: marker_id, + observed_reason: reason, + re_read: false + ) + unless gate.safe? + return audit_negative(row, gate.rationale, classification.recovery_class, now: now) + end + + # The pre-probe decision is only an exhaustion/in-flight gate. In + # particular, do not reject attempt two for the *old* fingerprint + # before a changed cheap health signal gets its fresh probe. + decision = Policy.decide(record: record, healthy_fingerprint: nil, now: now) + if decision.action == :refuse + if decision.rationale == "exhausted" + begin + @state.mark_exhausted!(project: project, task_key: task_key, + stage: stage, reason: reason) + rescue State::Suspended + nil + end + end + return audit_negative(row, decision.rationale, classification.recovery_class, now: now) + end + + signal = health_signal(classification, config, row) + signal_changed = !signal.to_s.empty? && signal.to_s != record["last_probe_signal"].to_s + need_probe = record["last_probe_at"].to_s.empty? || signal_changed || + Policy.should_reprobe?(record: record, now: now) + health_result = nil + if need_probe + cache_hit = health_cache_hit?(classification, config, row) + if !cache_hit && @probes_this_tick >= Policy::MAX_UNIQUE_PROBES_PER_TICK + return audit_negative(row, "probe_budget_deferred", classification.recovery_class, now: now) + end + @probes_this_tick += 1 unless cache_hit + health_result = @health.probe( + recovery_class: classification.recovery_class, + config: config, + project_root: project_root_for(row), + stage: stage + ) + begin + @state.upsert!( + project: project, task_key: task_key, stage: stage, reason: reason, + last_probe_at: now, + last_probe_fingerprint: health_result.fingerprint, + last_probe_healthy: health_result.healthy, + last_probe_rationale: health_result.rationale, + last_probe_signal: signal, + marker_id: marker_id, + phase: record["phase"] || "candidate" + ) + rescue State::Suspended => e + @logger&.event(:auto_retry_state_error, message: e.message) + return + end + unless health_result.healthy + return audit_negative(row, health_result.rationale || "probe_unhealthy", + classification.recovery_class, now: now, + extra: { probes: health_result.probes, + correlation_id: health_result.correlation_id }) + end + else + unless record["last_probe_healthy"] == true + return audit_negative( + row, + record["last_probe_rationale"].to_s.empty? ? "probe_unhealthy_cached" : record["last_probe_rationale"], + classification.recovery_class, + now: now, + extra: { probe_cached: true, next_probe_at: next_probe_at(record) } + ) + end + health_result = Health::ProbeSetResult.new( + healthy: true, + fingerprint: record["last_probe_fingerprint"], + probes: [], + rationale: "cached", + correlation_id: nil + ) + end + + decision = Policy.decide( + record: @state.get(project: project, task_key: task_key, stage: stage, reason: reason) || record, + healthy_fingerprint: health_result.fingerprint, + now: now + ) + + case decision.action + when :wait + begin + rec = @state.get(project: project, task_key: task_key, stage: stage, reason: reason) || {} + if rec["second_candidate_fingerprint"].to_s != health_result.fingerprint.to_s + @state.upsert!( + project: project, task_key: task_key, stage: stage, reason: reason, + second_candidate_fingerprint: health_result.fingerprint, + second_candidate_first_seen_at: now, + eligible_at: decision.eligible_at + ) + end + rescue State::Suspended + nil + end + audit_negative(row, decision.rationale, classification.recovery_class, now: now, + extra: { eligible_at: decision.eligible_at&.utc&.iso8601 }) + when :retry, :probe + return if decision.action == :probe && !health_result.healthy + + # Re-check safety immediately before queueing clear. + gate2 = safety.check(row: row, observed_marker_id: marker_id, + observed_reason: reason, re_read: true) + unless gate2.safe? + return audit_negative(row, gate2.rationale, classification.recovery_class, now: now) + end + + dispatch_clear_and_retry( + row: row, classification: classification, task_key: task_key, + health_result: health_result, reason: reason, marker_id: marker_id, + now: now + ) + else + audit_negative(row, decision.rationale, classification.recovery_class, now: now) + end + end + + def dispatch_clear_and_retry(row:, classification:, task_key:, health_result:, + reason:, marker_id:, now:) + project = row_get(row, :project).to_s + slug = row_get(row, :slug).to_s + stage = row_get(row, :stage).to_s + attrs = row_get(row, :marker_attrs) || {} + match_attr = Hive::RecoverySequence.match_attr_for_error(attrs) + match_attr ||= "marker_id=#{marker_id},reason=#{reason}" if !marker_id.empty? + + recovery_id = "ar-#{SecureRandom.hex(8)}" + clear_rid = request_id_for(recovery_id, "clear") + retry_rid = request_id_for(recovery_id, "rerun") + commands = Hive::RecoverySequence.commands( + project: project, + slug: slug, + stage: stage, + marker: "ERROR", + match_attr: match_attr, + workflow: row_get(row, :workflow) + ) + return audit_negative(row, "no_retry_verb", classification.recovery_class, now: now) if commands.empty? + + clear_argv = commands[0] + # Attach internal recovery id for auto-clear path. + if clear_argv && clear_argv[1] == "markers" + clear_argv = clear_argv + [ "--recovery-id", recovery_id ] + end + rerun_argv = commands[1] || commands[0] + + if @dry_run + audit(project: project, slug: slug, stage: stage, + action: "dry_run_queue", rationale: "would_clear_and_retry", + marker_reason: reason, recovery_class: classification.recovery_class, + task_id: row_get(row, :id), marker_id: marker_id, + recovery_id: recovery_id, fingerprint: health_result.fingerprint, + probes: health_result.probes, + correlation_id: health_result.correlation_id) + return + end + + begin + @state.upsert!( + project: project, task_key: task_key, stage: stage, reason: reason, + phase: "clear_queued", + recovery_id: recovery_id, + recovery_class: classification.recovery_class, + slug: slug, + folder: row_get(row, :folder), + marker_id: marker_id, + last_probe_fingerprint: health_result.fingerprint, + pending_fingerprint: health_result.fingerprint, + clear_argv: clear_argv, + rerun_argv: rerun_argv, + clear_request_id: clear_rid, + retry_request_id: retry_rid + ) + rescue State::Suspended => e + @logger&.event(:auto_retry_state_error, message: e.message) + return + end + + # Publish the continuation sidecar before the runnable clear request. + # A crash can therefore leave an inert sidecar, never a clear-only + # request that strands a markerless task. + if rerun_argv && rerun_argv != clear_argv + begin + @request_queue.write_sequence!(clear_rid, remaining_argvs: [ rerun_argv ], + state_home: state_home_for_queue, + recovery_id: recovery_id) + rescue StandardError => e + @logger&.event(:auto_retry_sequence_error, message: e.message) + return + end + end + + published = ensure_request!( + project: project, slug: slug, argv: clear_argv, + recovery_id: recovery_id, recovery_step: "clear", + request_id: clear_rid + ) + return unless published + + audit(project: project, slug: slug, stage: stage, + action: "queue_clear_and_retry", + rationale: "healthy_and_safe", + marker_reason: reason, + recovery_class: classification.recovery_class, + task_id: row_get(row, :id), + marker_id: marker_id, + recovery_id: recovery_id, + request_id: clear_rid, + fingerprint: health_result.fingerprint, + attempt: ((@state.get(project: project, task_key: task_key, stage: stage, reason: reason) || {})["attempts_dispatched"].to_i + 1), + max_attempts: Policy::MAX_ATTEMPTS, + probes: health_result.probes, + correlation_id: health_result.correlation_id, + folder: row_get(row, :folder)) + end + + def reconcile_inflight(row, record, task_key, now:) + project = row_get(row, :project).to_s + slug = row_get(row, :slug).to_s + stage = row_get(row, :stage).to_s + reason = record["reason"] || (row_get(row, :marker_attrs) || {})["reason"] + phase = record["phase"].to_s + @logger&.event(:auto_retry_reconcile, + project: project, slug: slug, stage: stage, + phase: phase, recovery_id: record["recovery_id"]) + + marker = row_get(row, :marker).to_s.downcase + attrs = row_get(row, :marker_attrs) || {} + current_reason = (attrs["reason"] || attrs[:reason]).to_s + current_marker_id = (attrs["marker_id"] || attrs[:marker_id]).to_s + same_marker = marker == "error" && current_reason == reason.to_s && + (record["marker_id"].to_s.empty? || + current_marker_id == record["marker_id"].to_s) + + if phase == "retry_dispatched" + return if @controller&.running_task?(project: project, slug: slug) + # Accounting is persisted before spawn. If spawn then fails, the + # dispatcher releases the claim back to the same durable request. + # Keep the idempotent dispatch transition live until that exact + # request is successfully claimed/spawned and removed on reap. + return if request_exists?(record["retry_request_id"]) + + if marker == "error" && current_reason == reason.to_s && !same_marker + terminal = record["attempts_dispatched"].to_i >= Policy::MAX_ATTEMPTS ? "exhausted" : "failed" + transition_record!(record, phase: terminal, marker_id: current_marker_id) + audit_terminal(row, record, terminal, now: now) + elsif terminal_success_marker?(marker) || stage != record["stage"].to_s + transition_record!(record, phase: "succeeded") + audit_terminal(row, record, "succeeded", now: now) + elsif marker == "error" || marker == "none" + transition_record!(record, phase: "aborted") + audit_terminal(row, record, "aborted", now: now) + else + transition_record!(record, phase: "aborted") + audit_terminal(row, record, "aborted", now: now) + end + return + end + + if same_marker + publish_clear_from_record(record) + elsif marker == "none" + publish_rerun_from_record(record) + elsif terminal_success_marker?(marker) + transition_record!(record, phase: "succeeded") + else + transition_record!(record, phase: "aborted") + end + end + + def ensure_request!(project:, slug:, argv:, recovery_id:, recovery_step:, request_id: nil) + request_id ||= request_id_for(recovery_id, recovery_step) + return request_id if request_exists?(request_id) + + @request_queue.write_request!( + project: project, + slug: slug, + argv: argv, + requestor: "auto_retry", + trigger: "auto_retry:#{recovery_step}", + request_id: request_id, + recovery_id: recovery_id, + recovery_step: recovery_step + ) + request_id + rescue StandardError => e + @logger&.event(:auto_retry_queue_error, message: e.message) + nil + end + + def request_id_for(recovery_id, recovery_step) + ::Digest::SHA256.hexdigest("#{recovery_id}:#{recovery_step}")[0, 16] + end + + def request_exists?(request_id) + if @request_queue.respond_to?(:request_exists?) + return @request_queue.request_exists?(request_id, state_home: state_home_for_queue) + end + + Array(@request_queue.pending).any? { |request| request.request_id.to_s == request_id.to_s } + rescue StandardError + false + end + + def publish_clear_from_record(record) + return abort_incoherent_record(record) unless coherent_recovery_record?(record) + + clear_id = record["clear_request_id"].to_s + rerun = Array(record["rerun_argv"]) + @request_queue.write_sequence!( + clear_id, + remaining_argvs: [ rerun ], + state_home: state_home_for_queue, + recovery_id: record["recovery_id"] + ) + ensure_request!( + project: record["project"], slug: record["slug"], + argv: Array(record["clear_argv"]), recovery_id: record["recovery_id"], + recovery_step: "clear", request_id: clear_id + ) + rescue StandardError => e + @logger&.event(:auto_retry_reconcile, phase: record["phase"], + recovery_id: record["recovery_id"], error_class: e.class.name, + message: e.message) + false + end + + def publish_rerun_from_record(record) + return abort_incoherent_record(record) unless coherent_recovery_record?(record) + + transition_record!(record, phase: "retry_queued") + ensure_request!( + project: record["project"], slug: record["slug"], + argv: Array(record["rerun_argv"]), recovery_id: record["recovery_id"], + recovery_step: "rerun", request_id: record["retry_request_id"] + ) + end + + def coherent_recovery_record?(record) + !record["project"].to_s.empty? && !record["slug"].to_s.empty? && + !record["stage"].to_s.empty? && !record["reason"].to_s.empty? && + !record["recovery_id"].to_s.empty? && + !record["clear_request_id"].to_s.empty? && + !record["retry_request_id"].to_s.empty? && + Array(record["clear_argv"]).any? && Array(record["rerun_argv"]).any? + end + + def abort_incoherent_record(record) + transition_record!(record, phase: "aborted", terminal_rationale: "incoherent_recovery_record") + false + rescue State::Error => e + @logger&.event(:auto_retry_state_error, message: e.message) + false + end + + def transition_record!(record, phase:, **fields) + @state.upsert!( + project: record.fetch("project"), task_key: record.fetch("task_key"), + stage: record.fetch("stage"), reason: record.fetch("reason"), + phase: phase, **fields + ) + end + + def inflight_record_for(project:, task_key:, stage:) + @state.load_all.values.find do |record| + record.is_a?(Hash) && record["project"].to_s == project.to_s && + record["task_key"].to_s == task_key.to_s && + record["stage"].to_s == stage.to_s && + %w[clear_queued marker_cleared retry_queued retry_dispatched].include?(record["phase"].to_s) + end + end + + def reconcile_records_without_current_stage(rows, now:) + @state.load_all.values.each do |record| + next unless record.is_a?(Hash) + next unless %w[clear_queued marker_cleared retry_queued retry_dispatched].include?(record["phase"].to_s) + + row = rows.find { |candidate| row_matches_record?(candidate, record) } + # A temporarily absent row is not authoritative terminal state. + next unless row + next if row_get(row, :stage).to_s == record["stage"].to_s + + terminal = record["phase"].to_s == "retry_dispatched" ? "succeeded" : "aborted" + transition_record!(record, phase: terminal) + audit_terminal(row, record, terminal, now: now) if row + end + rescue State::Error => e + @logger&.event(:auto_retry_state_error, message: e.message) + end + + def row_matches_record?(row, record) + return false unless row_get(row, :project).to_s == record["project"].to_s + + slug_match = !record["slug"].to_s.empty? && row_get(row, :slug).to_s == record["slug"].to_s + id_match = !record["task_key"].to_s.empty? && row_get(row, :id).to_s == record["task_key"].to_s + slug_match || id_match + end + + def fair_rows(rows) + return rows if rows.empty? + + rows.rotate(@probe_cursor % rows.length) + end + + def advance_probe_cursor(row_count) + return if row_count.zero? + + @probe_cursor = (@probe_cursor + Policy::MAX_UNIQUE_PROBES_PER_TICK) % row_count + end + + def health_signal(classification, config, row) + return nil unless @health.respond_to?(:signal_fingerprint) + + @health.signal_fingerprint( + recovery_class: classification.recovery_class, + config: config, + project_root: project_root_for(row), + stage: row_get(row, :stage) + ) + end + + def health_cache_hit?(classification, config, row) + return false unless @health.respond_to?(:cached?) + + @health.cached?( + recovery_class: classification.recovery_class, + config: config, + project_root: project_root_for(row), + stage: row_get(row, :stage) + ) + end + + def next_probe_at(record) + last = Policy.send(:parse_time, record["last_probe_at"]) + last && (last + Policy::FALLBACK_REPROBE_SEC).utc.iso8601 + rescue StandardError + nil + end + + def terminal_success_marker?(marker) + %w[complete execute_complete review_complete waiting review_waiting execute_waiting].include?(marker.to_s) + end + + def audit_terminal(row, record, terminal, now:) + attrs = row ? (row_get(row, :marker_attrs) || {}) : {} + audit( + project: record["project"], slug: record["slug"], stage: record["stage"], + action: terminal, rationale: "authoritative_status", + marker_reason: record["reason"], recovery_class: record["recovery_class"], + folder: row && row_get(row, :folder), recovery_id: record["recovery_id"], + task_id: row ? row_get(row, :id) : record["task_key"], + marker_id: attrs["marker_id"] || attrs[:marker_id] || record["marker_id"], + fingerprint: record["last_attempted_fingerprint"] || record["last_probe_fingerprint"], + request_id: record["retry_request_id"], + attempt: record["attempts_dispatched"].to_i, + max_attempts: Policy::MAX_ATTEMPTS, observed_at: now.utc.iso8601 + ) + end + + def state_home_for_queue + Hive::Paths.state_home + end + + def project_root_for(row) + folder = row_get(row, :folder).to_s + return nil if folder.empty? + + # task folder is /.hive-state/stages// + parts = folder.split(File::SEPARATOR) + idx = parts.rindex(".hive-state") + return nil unless idx && idx.positive? + + File.join(*parts[0...idx]) + end + + def audit_negative(row, rationale, recovery_class, now:, extra: {}) + project = row_get(row, :project).to_s + slug = row_get(row, :slug).to_s + stage = row_get(row, :stage).to_s + attrs = row_get(row, :marker_attrs) || {} + reason = (attrs["reason"] || attrs[:reason]).to_s + task_key = State.task_key(task_id: row_get(row, :id), slug: slug) + sig = "#{rationale}|#{recovery_class}|#{attrs['marker_id']}" + record = @state.get(project: project, task_key: task_key, stage: stage, reason: reason) || {} + return unless Policy.should_emit_negative?(record: record, signature: sig, now: now) + + begin + @state.upsert!( + project: project, task_key: task_key, stage: stage, reason: reason, + last_negative_signature: sig, + last_negative_at: now + ) + rescue State::Suspended + nil + end + + facts = { + task_id: row_get(row, :id), + marker_id: attrs["marker_id"] || attrs[:marker_id], + fingerprint: record["last_probe_fingerprint"], + attempt: record["attempts_dispatched"].to_i, + max_attempts: Policy::MAX_ATTEMPTS, + recovery_id: record["recovery_id"], + phase: record["phase"] + }.merge(extra) + audit(project: project, slug: slug, stage: stage, + action: "refuse", rationale: rationale, + marker_reason: reason, recovery_class: recovery_class, + folder: row_get(row, :folder), + **facts) + end + + def audit(project:, slug:, stage:, action:, rationale:, marker_reason:, + recovery_class:, folder: nil, **extra) + payload = { + project: project, + slug: slug, + stage: stage, + action: action, + rationale: rationale, + marker_reason: marker_reason, + recovery_class: recovery_class&.to_s, + ts: @clock.call.utc.iso8601 + }.merge(extra) + + @logger&.event(:auto_retry_decision, **payload) + + return if folder.to_s.empty? + + begin + Hive::Events.emit( + task_folder: folder, + slug: slug, + stage: stage, + event_type: :auto_retry_decision, + agent: "daemon", + message: "#{action}:#{rationale}", + details: bound_details(payload) + ) + rescue ArgumentError + # events may not know the type until Unit 6 wires it + Hive::Events.emit( + task_folder: folder, + slug: slug, + stage: stage, + event_type: :error, + agent: "daemon", + message: "auto_retry #{action}:#{rationale}" + ) + rescue StandardError => e + @logger&.event(:auto_retry_audit_error, message: e.message) + end + end + + def bound_details(payload) + # Drop bulky probe excerpts for the task event surface. + payload = payload.dup + if payload[:probes] + payload[:probes] = Array(payload[:probes]).map do |p| + probe = {} + %w[name ok timed_out duration_sec exit_status error stdout stderr detail completion_record].each do |key| + value = if p.is_a?(Hash) && p.key?(key) + p[key] + elsif p.is_a?(Hash) && p.key?(key.to_sym) + p[key.to_sym] + end + probe[key] = value unless value.nil? + end + probe + end + end + payload + end + + def row_get(row, key) + if row.respond_to?(key) + row.public_send(key) + elsif row.is_a?(Hash) + row[key] || row[key.to_s] + end + end + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/evidence.rb b/lib/hive/daemon/auto_retry/evidence.rb new file mode 100644 index 00000000..f2ac1e1c --- /dev/null +++ b/lib/hive/daemon/auto_retry/evidence.rb @@ -0,0 +1,318 @@ +# frozen_string_literal: true + +require "yaml" +require "hive/diagnostic_helpers" +require "hive/diagnostic_evidence" +require "hive/secret_patterns" +require "hive/task_action" +require "hive/markers" + +module Hive + module Daemon + module AutoRetry + # Machine-oriented, fail-closed evidence loader for auto-retry + # classification. Reuses the containment/size/redaction discipline of + # DiagnosticEvidence and the marker_signature freshness gate shared + # with TaskAction diagnostics. + # + # Never raises to the daemon tick: every failure path returns a + # structured Hash with error/fresh flags so the classifier can emit + # a typed rejection rationale. + module Evidence + MAX_EVIDENCE_CHARS = 8_000 + LOG_EPISODE_WINDOW_SEC = 5 * 60 + EvidenceResult = Struct.new( + :text, :kind, :source_path, :fresh, :marker_signature_match, :error, + keyword_init: true + ) do + def to_h + { + text: text, + kind: kind, + source_path: source_path, + fresh: fresh, + marker_signature_match: marker_signature_match, + error: error + } + end + end + + module_function + + # @param folder [String] task folder + # @param state_file [String, nil] authoritative state file + # @param marker [Hive::Markers::Marker, Hash, nil] current marker + # @param marker_attrs [Hash, nil] + def load(folder:, state_file: nil, marker: nil, marker_attrs: nil) + root = folder.to_s + if root.strip.empty? || !File.directory?(root) + return failed(:missing_folder) + end + + expected_sig = compute_signature(marker, marker_attrs) + marker_summary = marker_summary_text(marker, marker_attrs, state_file) + + red = load_red_status(root, expected_sig, state_file) + return red if red + + log = load_latest_log( + root, + marker_summary, + expected_sig: expected_sig, + marker_attrs: marker_attrs, + state_file: state_file + ) + return log if log + + if marker_summary && !marker_summary.empty? + return EvidenceResult.new( + text: bound(marker_summary), + kind: :marker, + source_path: state_file, + fresh: true, + marker_signature_match: true, + error: nil + ) + end + + failed(:missing_diagnostic) + rescue StandardError, SystemStackError, NoMemoryError => e + failed(:evidence_error, detail: "#{e.class}: #{e.message}") + end + + def load_red_status(folder, expected_sig, state_file) + path = File.join(folder, "diagnostics", "red-status.md") + return nil unless File.file?(path) + return nil unless contained?(path, folder) + + body = safe_read(path) + return failed(:unreadable_diagnostic) if body.nil? + + frontmatter = parse_frontmatter(body) + sig = frontmatter["marker_signature"].to_s + sig_match = expected_sig.nil? || expected_sig.empty? || sig == expected_sig + return failed(:stale_diagnostic, kind: :red_status, source_path: path) unless sig_match + + if state_file && File.file?(state_file) + art_mtime = Hive::DiagnosticHelpers.safe_mtime(path) + state_mtime = Hive::DiagnosticHelpers.safe_mtime(state_file) + if art_mtime && state_mtime && art_mtime < state_mtime + return failed(:stale_diagnostic, kind: :red_status, source_path: path) + end + end + + text = frontmatter["summary"].to_s + text = body_first_line(body) if text.strip.empty? + text = body if text.strip.empty? + return nil if text.to_s.strip.empty? + + EvidenceResult.new( + text: bound(text), + kind: :red_status, + source_path: path, + fresh: true, + marker_signature_match: true, + error: nil + ) + end + private_class_method :load_red_status + + def load_latest_log(folder, marker_summary, expected_sig:, marker_attrs:, state_file:) + candidates = log_candidates(folder) + candidates.each do |path| + next unless contained?(path, folder) + next unless current_log_episode?( + path, + expected_sig: expected_sig, + marker_attrs: marker_attrs, + state_file: state_file + ) + + excerpt = Hive::DiagnosticHelpers.tail_file(path) + next if excerpt.to_s.strip.empty? + + parts = [ marker_summary, excerpt ].compact.reject { |p| p.to_s.strip.empty? } + text = parts.join(": ") + next if text.strip.empty? + + return EvidenceResult.new( + text: bound(text), + kind: :log, + source_path: path, + fresh: true, + marker_signature_match: true, + error: nil + ) + end + nil + end + private_class_method :load_latest_log + + def current_log_episode?(path, expected_sig:, marker_attrs:, state_file:) + tail = Hive::DiagnosticHelpers.tail_file(path) + marker_id = (marker_attrs || {})["marker_id"] || (marker_attrs || {})[:marker_id] + return true if !marker_id.to_s.empty? && tail.include?(marker_id.to_s) + return true if !expected_sig.to_s.empty? && tail.include?(expected_sig.to_s) + + return false unless state_file && File.file?(state_file) + + log_mtime = Hive::DiagnosticHelpers.safe_mtime(path) + state_mtime = Hive::DiagnosticHelpers.safe_mtime(state_file) + return false unless log_mtime && state_mtime + + (state_mtime - log_mtime).abs <= LOG_EPISODE_WINDOW_SEC + rescue SystemCallError, IOError + false + end + private_class_method :current_log_episode? + + def log_candidates(folder) + dirs = [ + File.join(folder, "logs"), + inferred_log_dir(folder) + ].compact.uniq + candidates = dirs.flat_map { |dir| Dir[File.join(dir, "*.log")] } + candidates.sort_by { |p| Hive::DiagnosticHelpers.safe_mtime(p) || Time.at(0) } + .last(Hive::DiagnosticHelpers::LOG_GLOB_CAP) + .reverse + rescue SystemCallError + [] + end + private_class_method :log_candidates + + def inferred_log_dir(folder) + require "hive/task" + match = Hive::Task::PATH_RE.match(folder.to_s) + return nil unless match + + File.join(match[:root], match[:state_dir], "logs", match[:slug]) + rescue LoadError, StandardError + nil + end + private_class_method :inferred_log_dir + + def contained?(path, folder) + real = File.realpath(path) + return false unless File.file?(real) + + roots = [ + Hive::DiagnosticHelpers.evidence_root_realpath(folder, trust_anchor: true), + Hive::DiagnosticHelpers.evidence_root_realpath(File.join(folder, "logs"), trust_anchor: false), + Hive::DiagnosticHelpers.evidence_root_realpath(inferred_log_dir(folder), trust_anchor: false) + ].compact + roots.any? { |root| Hive::DiagnosticHelpers.path_inside?(real, root) } + rescue SystemCallError + false + end + private_class_method :contained? + + def safe_read(path) + raw = File.open(path, "rb") do |f| + f.read(Hive::DiagnosticHelpers::FRONTMATTER_SCAN_BYTES + MAX_EVIDENCE_CHARS).to_s + end + Hive::DiagnosticHelpers.utf8(raw) + rescue SystemCallError, IOError + nil + end + private_class_method :safe_read + + def parse_frontmatter(body) + match = body.match(/\A---\n(.*?)\n---\n/m) + return {} unless match + + parsed = YAML.safe_load(match[1], permitted_classes: [ Time ]) || {} + return {} unless parsed.is_a?(Hash) + + parsed.transform_keys(&:to_s) + rescue Psych::Exception, SystemStackError, NoMemoryError + {} + end + private_class_method :parse_frontmatter + + def body_first_line(body) + in_frontmatter = false + body.each_line do |line| + stripped = line.strip + if stripped == "---" + in_frontmatter = !in_frontmatter + next + end + next if in_frontmatter || stripped.empty? + + return stripped + end + "" + end + private_class_method :body_first_line + + def compute_signature(marker, marker_attrs) + if marker.respond_to?(:name) && marker.respond_to?(:attrs) + return Hive::TaskAction.marker_signature(marker) + end + + attrs = marker_attrs + attrs = marker[:attrs] || marker["attrs"] if marker.is_a?(Hash) && attrs.nil? + name = if marker.is_a?(Hash) + marker[:name] || marker["name"] || :error + elsif marker.respond_to?(:name) + marker.name + else + :error + end + fake = Struct.new(:name, :attrs).new(name, attrs || {}) + Hive::TaskAction.marker_signature(fake) + rescue StandardError + nil + end + private_class_method :compute_signature + + def marker_summary_text(marker, marker_attrs, state_file) + if marker.respond_to?(:name) && !marker.respond_to?(:none?) + return Hive::Markers.summary(marker) + end + if marker.respond_to?(:none?) && !marker.none? + return Hive::Markers.summary(marker) + end + + if state_file && File.file?(state_file) + current = Hive::Markers.current(state_file) + return Hive::Markers.summary(current) unless current.none? + end + + attrs = marker_attrs || {} + reason = attrs["reason"] || attrs[:reason] + message = attrs["message"] || attrs[:message] + return nil if reason.nil? && (message.nil? || message.to_s.empty?) + + parts = [ "ERROR" ] + parts << "reason=#{reason}" if reason + parts << message.to_s if message && !message.to_s.empty? + parts.join(" ") + rescue StandardError + nil + end + private_class_method :marker_summary_text + + def bound(text) + redacted = Hive::SecretPatterns.redact( + Hive::DiagnosticHelpers.utf8(text.to_s).gsub(/\s+/, " ").strip + ) + Hive::DiagnosticHelpers.truncate(redacted, MAX_EVIDENCE_CHARS) + end + private_class_method :bound + + def failed(error, kind: nil, source_path: nil, detail: nil) + EvidenceResult.new( + text: detail, + kind: kind, + source_path: source_path, + fresh: error != :stale_diagnostic, + marker_signature_match: error != :stale_diagnostic, + error: error + ) + end + private_class_method :failed + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/health.rb b/lib/hive/daemon/auto_retry/health.rb new file mode 100644 index 00000000..c6939dea --- /dev/null +++ b/lib/hive/daemon/auto_retry/health.rb @@ -0,0 +1,524 @@ +# frozen_string_literal: true + +require "digest" +require "json" +require "stringio" +require "tmpdir" +require "fileutils" +require "hive/runtime_identity" +require "hive/daemon/auto_retry/probe_runner" +require "hive/stages/base" +require "hive/config" +require "hive/claude_launcher" +require "hive/commands/doctor" +require "hive/secret_patterns" + +module Hive + module Daemon + module AutoRetry + # Reason-specific health probes and canonical health fingerprints for + # auto-retry. Fail-closed: any timeout, nonzero exit, parse failure, + # missing binary, or identity mismatch is unhealthy. + class Health + STATUS_TIMEOUT = ProbeRunner::DEFAULT_STATUS_TIMEOUT_SEC + SMOKE_TIMEOUT = ProbeRunner::DEFAULT_SMOKE_TIMEOUT_SEC + + ProbeSetResult = Struct.new( + :healthy, :fingerprint, :probes, :rationale, :correlation_id, + keyword_init: true + ) + + def initialize(runner: nil, env: nil, clock: -> { Time.now }) + @runner = runner || ProbeRunner.new(env: scrubbed_env(env)) + @env = env + @clock = clock + @cache = {} + end + + # Reset per-tick cache (call at start of each daemon tick). + def reset_tick_cache! + @cache = {} + end + + def probe(recovery_class:, config:, project_root: nil, stage: nil) + key = cache_key(recovery_class, config, project_root, stage) + return @cache[key] if @cache.key?(key) + + result = case recovery_class.to_sym + when :codex_auth_401 + probe_codex(config, project_root: project_root) + when :claude_launcher + probe_claude(config, project_root: project_root, stage: stage) + else + ProbeSetResult.new( + healthy: false, fingerprint: nil, probes: [], + rationale: "unknown_recovery_class", + correlation_id: correlation_id + ) + end + @cache[key] = result + result + end + + def cached?(recovery_class:, config:, project_root: nil, stage: nil) + @cache.key?(cache_key(recovery_class, config, project_root, stage)) + end + + # Cheap, non-secret signal used to decide whether an unhealthy probe + # should run before the 30-minute fallback cadence. No subprocesses. + def signal_fingerprint(recovery_class:, config:, project_root: nil, stage: nil) + profile = profile_for(recovery_class, config, stage) + payload = { + "recovery_class" => recovery_class.to_s, + "runtime" => Hive::RuntimeIdentity.snapshot, + "config" => relevant_config_subset(config), + "project_root" => project_root.to_s, + "stage" => stage.to_s, + "agent_bin" => file_metadata(resolve_executable(profile&.bin)), + "wrapper" => file_metadata(claude_wrapper_path), + "credentials" => credential_metadata(recovery_class) + } + ::Digest::SHA256.hexdigest(JSON.generate(sort_keys(payload))) + rescue StandardError + nil + end + + def probe_codex(config, project_root: nil) + cid = correlation_id + probes = [] + profile = safe_profile(config, "execute") + bin = profile&.bin || "codex" + + doctor = required_doctor_green?(config, project_root) + probes << { "name" => "doctor_required", "ok" => doctor[:ok], + "detail" => doctor[:detail] } + return unhealthy(probes, cid, "doctor_unhealthy", config, profile) unless doctor[:ok] + + login = @runner.run([ bin, "login", "status" ], timeout_sec: STATUS_TIMEOUT) + probes << probe_entry("codex_login_status", login) + unless login.healthy? + return unhealthy(probes, cid, "codex_login_unhealthy", config, profile) + end + + smoke = run_codex_smoke(profile || safe_profile({}, "execute"), bin) + completion_ok = smoke.healthy? && codex_completion_record?(smoke.stdout) + smoke_entry = probe_entry("codex_exec_smoke", smoke) + smoke_entry["completion_record"] = completion_ok + probes << smoke_entry + unless smoke.healthy? + return unhealthy(probes, cid, "codex_smoke_unhealthy", config, profile) + end + unless completion_ok + return unhealthy(probes, cid, "codex_smoke_malformed", config, profile) + end + + fp = fingerprint( + recovery_class: :codex_auth_401, + config: config, + profile: profile, + extra: { + "login_ok" => true, + "smoke_ok" => true, + "doctor" => doctor_inventory(doctor) + } + ) + ProbeSetResult.new(healthy: true, fingerprint: fp, probes: probes, + rationale: "healthy", correlation_id: cid) + end + + def probe_claude(config, project_root: nil, stage: nil) + cid = correlation_id + probes = [] + profile = profile_for(:claude_launcher, config, stage) + failures = [] + + wrapper = claude_wrapper_path + wrapper_ok = wrapper && File.file?(wrapper) && File.executable?(wrapper) + probes << { + "name" => "claude_wrapper", + "ok" => wrapper_ok == true, + "detail" => wrapper_ok ? "present" : "missing_or_not_executable" + } + failures << "wrapper_unhealthy" unless wrapper_ok + + readiness = readiness_self_check + probes << { "name" => "readiness_detector", "ok" => readiness[:ok], + "detail" => readiness[:detail] } + failures << "readiness_unhealthy" unless readiness[:ok] + + tmux_probe, tmux_row = bounded_tmux_row + tmux_ok = tmux_row[:status].to_s == "present" + probes << probe_entry("tmux", tmux_probe).merge("detail" => tmux_row[:message]) + failures << "tmux_unhealthy" unless tmux_ok + + version = agent_version(profile) + probes << version[:entry] + failures << "claude_runtime_unhealthy" unless version[:ok] + + doctor = required_doctor_green?(config, project_root, dependency_rows: [ tmux_row ]) + probes << { "name" => "doctor_required", "ok" => doctor[:ok], + "detail" => doctor[:detail] } + failures << "doctor_unhealthy" unless doctor[:ok] + + identity = identity_matches? + probes << { "name" => "runtime_identity", "ok" => identity[:ok], + "detail" => identity[:detail] } + failures << "identity_mismatch" unless identity[:ok] + + if failures.any? + return ProbeSetResult.new( + healthy: false, fingerprint: nil, probes: probes, + rationale: failures.first, correlation_id: cid + ) + end + + fp = fingerprint( + recovery_class: :claude_launcher, + config: config, + profile: profile, + extra: { + "wrapper_digest" => file_digest(wrapper), + "wrapper_mtime" => safe_mtime_i(wrapper), + "readiness" => readiness[:detail], + "tmux" => tmux_row[:message].to_s, + "claude_version" => version[:version], + "doctor" => "green", + "identity" => Hive::RuntimeIdentity.snapshot + } + ) + ProbeSetResult.new(healthy: true, fingerprint: fp, probes: probes, + rationale: "healthy", correlation_id: cid) + end + + # Lightweight in-process Doctor view: only required agent skill / + # dependency rows. Optional warnings (qmd, legacy runtime) do not fail. + def self.required_agent_report(config:, project_root: nil, dependency_rows: nil) + required = Hive::Commands::Doctor.required_agent_rows( + config: config, + project_root: project_root, + dependency_rows: dependency_rows + ) + failing = Array(required).select do |r| + status = (r[:status] || r["status"]).to_s + %w[missing version_too_old].include?(status) + end + { + ok: failing.empty?, + rows: required, + failing: failing, + detail: failing.empty? ? "green" : failing.map { |r| r[:label] || r["label"] }.join(",") + } + rescue StandardError => e + { ok: false, rows: [], failing: [], detail: "#{e.class}: #{e.message}" } + end + + def fingerprint(recovery_class:, config:, profile: nil, extra: {}) + payload = { + "recovery_class" => recovery_class.to_s, + "version" => Hive::RuntimeIdentity.version, + "code_fingerprint" => Hive::RuntimeIdentity.code_fingerprint, + "binary_path" => Hive::RuntimeIdentity.binary_path, + "agent" => profile&.name.to_s, + "agent_bin" => profile&.bin.to_s, + "agent_binary" => file_metadata(resolve_executable(profile&.bin)), + "credentials" => credential_metadata(recovery_class), + "config" => relevant_config_subset(config), + "extra" => stringify_keys(extra) + } + canonical = JSON.generate(sort_keys(payload)) + # Ensure secrets never enter the fingerprint source. + redacted = Hive::SecretPatterns.redact(canonical) + ::Digest::SHA256.hexdigest(redacted) + end + + private + + def run_codex_smoke(profile, bin) + Dir.mktmpdir("hive-codex-smoke-") do |dir| + # Tiny isolated exec; never the task worktree. + prompt = "Reply with exactly: ok" + argv = [ bin, profile&.headless_flag || "exec" ] + argv.concat(profile&.output_format_flags || [ "--json" ]) + argv.concat([ "--skip-git-repo-check", "-" ]) + @runner.run( + argv, + timeout_sec: SMOKE_TIMEOUT, + chdir: dir, + stdin_data: "#{prompt}\n" + ) + end + end + + def codex_completion_record?(stdout) + stdout.to_s.each_line.any? do |line| + event = JSON.parse(line) + %w[turn.completed response.completed result].include?(event["type"].to_s) && + !%w[failed error cancelled].include?((event["status"] || event.dig("turn", "status")).to_s) + rescue JSON::ParserError + false + end + end + + def claude_wrapper_path + Hive::ClaudeLauncher.interactive_wrapper_path + end + + def readiness_self_check + Hive::ClaudeLauncher.readiness_detector_self_check + end + + def required_doctor_green?(config, project_root, dependency_rows: nil) + rows = dependency_rows + if rows.nil? && Hive::Config.claude_mode(config) == :tmux + _probe, row = bounded_tmux_row + rows = [ row ] + end + self.class.required_agent_report( + config: config, + project_root: project_root, + dependency_rows: rows + ) + end + + def identity_matches? + snap = Hive::RuntimeIdentity.snapshot + if snap["code_fingerprint"].nil? + return { ok: false, detail: "no_code_fingerprint" } + end + + hive_bin = Hive::RuntimeIdentity.binary_path + return { ok: false, detail: "no_binary_path" } if hive_bin.to_s.empty? + + result = @runner.run([ hive_bin, "version", "--json" ], timeout_sec: STATUS_TIMEOUT) + unless result.healthy? + return { ok: false, detail: "cli_probe_failed" } + end + + parsed = JSON.parse(result.stdout) + return { ok: false, detail: "cli_identity_not_object" } unless parsed.is_a?(Hash) + + cli_version = parsed["version"] + cli_fp = parsed["code_fingerprint"] + cli_bin = parsed["binary_path"] + if [ cli_version, cli_fp, cli_bin ].any? { |value| value.to_s.empty? } + return { ok: false, detail: "cli_identity_incomplete" } + end + ok = Hive::RuntimeIdentity.matches_cli?( + cli_version: cli_version, + cli_fingerprint: cli_fp, + cli_binary_path: cli_bin + ) + { ok: ok, detail: ok ? "match" : "mismatch" } + rescue StandardError => e + { ok: false, detail: e.class.name } + end + + def bounded_tmux_row + bin = Hive::ClaudeLauncher.tmux_bin + result = @runner.run([ bin, "-V" ], timeout_sec: STATUS_TIMEOUT) + status = "missing" + message = result.error.to_s + if result.healthy? + version = Hive::ClaudeLauncher.parse_tmux_version(result.stdout) + if version.nil? + message = "could not parse tmux -V output" + elsif (Hive::ClaudeLauncher.version_tuple(version) <=> + Hive::ClaudeLauncher.version_tuple(Hive::ClaudeLauncher::MIN_TMUX_VERSION)).negative? + status = "version_too_old" + message = "tmux #{version} below minimum #{Hive::ClaudeLauncher::MIN_TMUX_VERSION}" + else + status = "present" + message = "tmux #{version} found" + end + end + row = { + kind: "dependency", stage: "claude", label: "claude/tmux", + agent: "tmux", configured_skill: "tmux >= #{Hive::ClaudeLauncher::MIN_TMUX_VERSION}", + skill: "tmux", status: status, message: message + } + [ result, row ] + end + + def agent_version(profile) + unless profile + return { ok: false, version: nil, + entry: { "name" => "claude_version", "ok" => false, + "detail" => "profile_missing" } } + end + result = @runner.run([ profile.bin, profile.version_flag ], timeout_sec: STATUS_TIMEOUT) + version = result.stdout.to_s[/\d+\.\d+\.\d+/] + ok = result.healthy? && !version.nil? + if ok && profile.min_version + ok = (version.split(".").map(&:to_i) <=> + profile.min_version.split(".").map(&:to_i)) >= 0 + end + entry = probe_entry("claude_version", result) + entry["version"] = version + entry["ok"] = ok + { ok: ok, version: version, entry: entry } + end + + def doctor_inventory(report) + Array(report[:rows]).map do |row| + { + "label" => row[:label] || row["label"], + "status" => row[:status] || row["status"] + } + end + end + + def profile_for(recovery_class, config, stage) + if recovery_class.to_sym == :codex_auth_401 + safe_profile(config, "execute") + else + safe_profile(config, stage_role(stage)) || safe_profile(config, "plan") || safe_profile(config, "execute") + end + end + + def stage_role(stage) + stage.to_s.sub(/\A\d+-/, "").tr("-", "_") + end + + def resolve_executable(bin) + return nil if bin.to_s.empty? + return File.expand_path(bin) if bin.to_s.include?(File::SEPARATOR) + + ENV.fetch("PATH", "").split(File::PATH_SEPARATOR).each do |dir| + path = File.join(dir, bin.to_s) + return path if File.file?(path) && File.executable?(path) + end + nil + end + + def file_metadata(path) + return nil unless path && File.file?(path) + + stat = File.stat(path) + { "path" => File.realpath(path), "size" => stat.size, "mtime" => stat.mtime.to_i } + rescue StandardError + nil + end + + def credential_metadata(recovery_class) + paths = if recovery_class.to_sym == :codex_auth_401 + [ File.expand_path("~/.codex/auth.json") ] + else + [ File.expand_path("~/.claude.json"), File.expand_path("~/.claude/.credentials.json") ] + end + paths.filter_map { |path| file_metadata(path) } + rescue StandardError + [] + end + + def relevant_config_subset(config) + return {} unless config.is_a?(Hash) + + { + "execute.agent" => config.dig("execute", "agent"), + "brainstorm.agent" => config.dig("brainstorm", "agent"), + "plan.agent" => config.dig("plan", "agent"), + "claude.mode" => config.dig("claude", "mode"), + "agents.codex.bin" => config.dig("agents", "codex", "bin"), + "agents.claude.bin" => config.dig("agents", "claude", "bin") + }.compact + end + + def safe_profile(config, stage) + Hive::Stages::Base.stage_profile(config, stage) + rescue StandardError + nil + end + + def probe_entry(name, result) + { + "name" => name, + "ok" => result.healthy?, + "timed_out" => result.timed_out, + "exit_status" => result.exit_status, + "duration_sec" => result.duration_sec, + "error" => result.error, + "stdout" => truncate(result.stdout, 400), + "stderr" => truncate(result.stderr, 400) + } + end + + def unhealthy(probes, cid, rationale, config, profile) + ProbeSetResult.new( + healthy: false, + fingerprint: nil, + probes: probes, + rationale: rationale, + correlation_id: cid + ) + end + + def truncate(text, max) + s = text.to_s + s.bytesize <= max ? s : s.byteslice(0, max).to_s.scrub + "…" + end + + def file_digest(path) + return nil unless path && File.file?(path) + + ::Digest::SHA256.file(path).hexdigest + rescue StandardError + nil + end + + def safe_mtime_i(path) + File.mtime(path).to_i + rescue StandardError + nil + end + + def correlation_id + ::Digest::SHA256.hexdigest("#{@clock.call.to_f}-#{object_id}")[0, 12] + end + + def cache_key(recovery_class, config, project_root, stage = nil) + [ + recovery_class.to_s, + relevant_config_subset(config).inspect, + project_root.to_s, + stage.to_s + ].join("|") + end + + def scrubbed_env(env) + base = (env || ENV).to_h + # Drop known secret-bearing keys from child environment copies used + # only for probes; keep PATH/HOME/USER and agent auth files intact + # via the normal process environment when env is nil. + return nil if env.nil? + + base.reject { |k, _| k.to_s.match?(/TOKEN|SECRET|PASSWORD|API_KEY|CREDENTIAL/i) } + end + + def sort_keys(obj) + case obj + when Hash + obj.keys.map(&:to_s).sort.each_with_object({}) do |k, acc| + value = obj.key?(k) ? obj[k] : obj[k.to_sym] + acc[k] = sort_keys(value) + end + when Array + obj.map { |v| sort_keys(v) } + else + obj + end + end + + def stringify_keys(obj) + case obj + when Hash + obj.each_with_object({}) { |(k, v), acc| acc[k.to_s] = stringify_keys(v) } + when Array + obj.map { |v| stringify_keys(v) } + else + obj + end + end + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/policy.rb b/lib/hive/daemon/auto_retry/policy.rb new file mode 100644 index 00000000..0837e015 --- /dev/null +++ b/lib/hive/daemon/auto_retry/policy.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Hive + module Daemon + module AutoRetry + # Hardcoded conservative v1 policy constants and pure decision logic + # over durable state records. + module Policy + MAX_ATTEMPTS = 2 + SECOND_ATTEMPT_BACKOFF_SEC = 30 * 60 + FALLBACK_REPROBE_SEC = 30 * 60 + NEGATIVE_DECISION_THROTTLE_SEC = 30 * 60 + MAX_UNIQUE_PROBES_PER_TICK = 2 + + module_function + + Decision = Struct.new( + :action, :rationale, :eligible_at, :attempt_number, + keyword_init: true + ) + + # Decide whether a probe/retry may proceed for a state record given + # a newly observed healthy fingerprint (or nil when not yet probed). + # + # @param record [Hash] durable state record + # @param healthy_fingerprint [String, nil] + # @param now [Time] + def decide(record:, healthy_fingerprint: nil, now: Time.now) + record = stringify(record) + attempts = record["attempts_dispatched"].to_i + phase = record["phase"].to_s + + if phase == "exhausted" || attempts >= MAX_ATTEMPTS + return Decision.new(action: :refuse, rationale: "exhausted", + eligible_at: nil, attempt_number: attempts) + end + + if %w[clear_queued marker_cleared retry_queued retry_dispatched].include?(phase) + return Decision.new(action: :reconcile, rationale: "in_flight_#{phase}", + eligible_at: nil, attempt_number: attempts + 1) + end + + if healthy_fingerprint.nil? || healthy_fingerprint.to_s.empty? + return Decision.new(action: :probe, rationale: "need_health_probe", + eligible_at: now, attempt_number: attempts + 1) + end + + last_fp = record["last_attempted_fingerprint"].to_s + if attempts.zero? + # First retry: immediately eligible after first healthy signal. + if last_fp.empty? || last_fp != healthy_fingerprint.to_s || + record["last_probe_healthy"] != true + return Decision.new(action: :retry, rationale: "first_healthy_signal", + eligible_at: now, attempt_number: 1) + end + # Same fingerprint already tried? Should not happen with attempts=0. + return Decision.new(action: :retry, rationale: "first_healthy_signal", + eligible_at: now, attempt_number: 1) + end + + # Attempt 2: fingerprint must differ from attempt 1 and remain the + # healthy candidate for 30 minutes, then pass a fresh probe. + if healthy_fingerprint.to_s == last_fp + return Decision.new(action: :refuse, rationale: "unchanged_fingerprint", + eligible_at: nil, attempt_number: attempts + 1) + end + + candidate_fp = record["second_candidate_fingerprint"].to_s + candidate_since = parse_time(record["second_candidate_first_seen_at"]) + if candidate_fp != healthy_fingerprint.to_s || candidate_since.nil? + # New candidate window starts now; caller persists it. + eligible = now + SECOND_ATTEMPT_BACKOFF_SEC + return Decision.new(action: :wait, rationale: "second_attempt_backoff", + eligible_at: eligible, attempt_number: 2) + end + + eligible_at = candidate_since + SECOND_ATTEMPT_BACKOFF_SEC + if now < eligible_at + return Decision.new(action: :wait, rationale: "second_attempt_backoff", + eligible_at: eligible_at, attempt_number: 2) + end + + Decision.new(action: :retry, rationale: "second_attempt_ready", + eligible_at: now, attempt_number: 2) + end + + def should_reprobe?(record:, now: Time.now) + record = stringify(record) + last = parse_time(record["last_probe_at"]) + return true if last.nil? + + (now - last) >= FALLBACK_REPROBE_SEC + end + + def should_emit_negative?(record:, signature:, now: Time.now) + record = stringify(record) + last_sig = record["last_negative_signature"].to_s + last_at = parse_time(record["last_negative_at"]) + return true if last_sig != signature.to_s + return true if last_at.nil? + + (now - last_at) >= NEGATIVE_DECISION_THROTTLE_SEC + end + + def stringify(record) + return {} if record.nil? + return record.transform_keys(&:to_s) if record.is_a?(Hash) + + {} + end + private_class_method :stringify + + def parse_time(value) + return value if value.is_a?(Time) + return nil if value.nil? || value.to_s.empty? + + Time.parse(value.to_s) + rescue ArgumentError + nil + end + private_class_method :parse_time + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/probe_runner.rb b/lib/hive/daemon/auto_retry/probe_runner.rb new file mode 100644 index 00000000..1f696dd2 --- /dev/null +++ b/lib/hive/daemon/auto_retry/probe_runner.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +require "open3" +require "hive/secret_patterns" +require "hive/diagnostic_helpers" + +module Hive + module Daemon + module AutoRetry + # Bounded array-argv subprocess runner for auto-retry health probes. + # Uses a process group + monotonic deadline; on timeout TERM → grace → + # KILL the group. Captures stdout/stderr separately, caps excerpts, + # scrubs invalid UTF-8, and redacts secrets before returning. + class ProbeRunner + DEFAULT_STATUS_TIMEOUT_SEC = 10 + DEFAULT_SMOKE_TIMEOUT_SEC = 30 + DEFAULT_TERM_GRACE_SEC = 1.0 + DEFAULT_EXCERPT_BYTES = 2_048 + + Result = Struct.new( + :ok, :exit_status, :timed_out, :signaled, :stdout, :stderr, + :duration_sec, :error, :argv, keyword_init: true + ) do + def healthy? + ok == true + end + end + + def initialize(env: nil, excerpt_bytes: DEFAULT_EXCERPT_BYTES, + term_grace_sec: DEFAULT_TERM_GRACE_SEC) + @env = env + @excerpt_bytes = excerpt_bytes + @term_grace_sec = term_grace_sec + end + + # @param argv [Array] command + args (no shell) + # @param timeout_sec [Numeric] + # @param chdir [String, nil] + # @param stdin_data [String, nil] + def run(argv, timeout_sec:, chdir: nil, stdin_data: nil) + argv = Array(argv).map(&:to_s) + if argv.empty? || argv.any?(&:empty?) + return Result.new(ok: false, error: "empty_argv", argv: argv, + exit_status: nil, timed_out: false, signaled: false, + stdout: "", stderr: "", duration_sec: 0.0) + end + + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + out_r, out_w = IO.pipe + err_r, err_w = IO.pipe + pid = nil + out_thread = nil + err_thread = nil + in_thread = nil + in_r = nil + in_w = nil + + spawn_opts = { + out: out_w, + err: err_w, + pgroup: true, + close_others: true + } + spawn_opts[:chdir] = chdir if chdir + if stdin_data + in_r, in_w = IO.pipe + spawn_opts[:in] = in_r + end + + pid = if @env + Process.spawn(@env, *argv, spawn_opts) + else + Process.spawn(*argv, spawn_opts) + end + out_w.close + err_w.close + stdout = "" + stderr = "" + out_thread = capture_thread(out_r) { |captured| stdout = captured } + err_thread = capture_thread(err_r) { |captured| stderr = captured } + if stdin_data + in_r.close + in_thread = Thread.new do + begin + in_w.write(stdin_data.to_s) + rescue Errno::EPIPE, IOError + nil + ensure + in_w.close unless in_w.closed? + end + end + end + + timed_out = false + status = nil + deadline = start + timeout_sec.to_f + loop do + captured = Process.wait2(pid, Process::WNOHANG) + if captured + status = captured.last + break + end + if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + timed_out = true + status = terminate_group(pid) + break + end + sleep 0.02 + end + + in_thread&.join(0.5) + finish_capture(out_thread, out_r) + finish_capture(err_thread, err_r) + duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + + exit_status = status&.exitstatus + signaled = status && !status.exited? && status.signaled? + ok = !timed_out && !signaled && status&.success? + + Result.new( + ok: ok == true, + exit_status: exit_status, + timed_out: timed_out, + signaled: signaled == true, + stdout: stdout, + stderr: stderr, + duration_sec: duration.round(3), + error: timed_out ? "timeout" : (signaled ? "signaled" : (ok ? nil : "nonzero_exit")), + argv: argv + ) + rescue Errno::ENOENT => e + Result.new(ok: false, error: "missing_executable", exit_status: nil, + timed_out: false, signaled: false, stdout: "", + stderr: redact(e.message), duration_sec: 0.0, argv: argv) + rescue StandardError => e + Result.new(ok: false, error: "#{e.class}: #{e.message}", exit_status: nil, + timed_out: false, signaled: false, stdout: "", + stderr: redact(e.message), duration_sec: 0.0, argv: argv) + ensure + [ out_r, out_w, err_r, err_w, in_r, in_w ].each { |io| close_quietly(io) } + end + + private + + def close_quietly(io) + io&.close unless io&.closed? + rescue StandardError + nil + end + + def terminate_group(pid) + return nil unless pid + + begin + Process.kill("-TERM", pid) + rescue Errno::ESRCH, Errno::EPERM + return wait_status(pid) + end + deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @term_grace_sec + loop do + begin + captured = Process.wait2(pid, Process::WNOHANG) + return captured.last if captured + rescue Errno::ECHILD + return nil + end + break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline + + sleep 0.05 + end + begin + Process.kill("-KILL", pid) + rescue Errno::ESRCH, Errno::EPERM + nil + end + wait_status(pid) + end + + def wait_status(pid) + Process.wait2(pid).last + rescue Errno::ECHILD + nil + end + + def capture_thread(io, &block) + Thread.new { block.call(read_capped(io)) }.tap do |thread| + thread.report_on_exception = false + end + end + + def finish_capture(thread, io) + return unless thread + return if thread.join(1) + + io.close unless io.closed? + thread.join(0.2) + thread.kill if thread.alive? + rescue IOError + nil + end + + def read_capped(io) + raw = +"" + while raw.bytesize < @excerpt_bytes + chunk = io.read([ 512, @excerpt_bytes - raw.bytesize ].min) + break if chunk.nil? || chunk.empty? + + raw << chunk + end + # Drain remainder without retaining + begin + while io.read(4096) + # discard + end + rescue StandardError + nil + end + redact(Hive::DiagnosticHelpers.utf8(raw)) + rescue StandardError + "" + end + + def redact(text) + Hive::SecretPatterns.redact(Hive::DiagnosticHelpers.utf8(text.to_s)) + end + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/safety.rb b/lib/hive/daemon/auto_retry/safety.rb new file mode 100644 index 00000000..b3e3a6e8 --- /dev/null +++ b/lib/hive/daemon/auto_retry/safety.rb @@ -0,0 +1,241 @@ +# frozen_string_literal: true + +require "hive/brainstorm_parser" +require "hive/daemon/auto_retry/probe_runner" +require "hive/markers" +require "hive/lock" +require "hive/task" + +module Hive + module Daemon + module AutoRetry + # Fail-closed stage-specific replay-safety guards. Run before probes + # and again immediately before marker clear. + class Safety + Result = Struct.new(:safe, :rationale, keyword_init: true) do + def safe? + safe == true + end + end + + GENERATED_MARKER_LINE = /\A\s*#{Hive::Markers::MARKER_RE.source}\s*\z/m.freeze + + def initialize(controller: nil, enabled_projects: nil, legacy_projects: nil, + runner: nil) + @controller = controller + @enabled_projects = enabled_projects + @legacy_projects = legacy_projects || {} + @runner = runner || ProbeRunner.new + end + + # @param row [StatusConsumer::Row] + # @param observed_marker_id [String] + # @param observed_reason [String] + # @param re_read [Boolean] when true, re-read marker/task from disk + def check(row:, observed_marker_id:, observed_reason:, re_read: false) + project = row_get(row, :project).to_s + slug = row_get(row, :slug).to_s + stage = row_get(row, :stage).to_s + folder = row_get(row, :folder).to_s + + if @enabled_projects && @enabled_projects[project] == false + return unsafe(:project_disabled) + end + + if @legacy_projects[project] || (@legacy_projects.respond_to?(:[]) && @legacy_projects[project]) + return unsafe(:legacy_layout) + end + + if folder.empty? || !File.directory?(folder) + return unsafe(:missing_task_folder) + end + + if @controller&.running_task?(project: project, slug: slug) + return unsafe(:controller_owned) + end + + live_task_lock = if re_read + live_task_lock?(folder) + else + row_get(row, :live_task_lock) == true + end + lock_owned = row_get(row, :task_lock_owned) == true + if live_task_lock && !lock_owned + return unsafe(:live_task_lock) + end + + # Merge-watcher owns finalize recovery for allowlisted reasons. + if stage == "8-finalize" # coding-scoped: finalize recovery belongs to merge watcher + return unsafe(:merge_watcher_owned) + end + + unless %w[2-brainstorm 3-plan 4-execute].include?(stage) # coding-scoped: v1 replay proofs + return unsafe(:stage_not_supported) + end + + marker_ok = verify_marker(row, observed_marker_id, observed_reason, re_read: re_read) + return marker_ok unless marker_ok.safe? + + # coding-scoped (block): v1 stage-specific replay-safety policies + case stage + when "4-execute" + execute_safe?(row) + when "2-brainstorm" + brainstorm_safe?(row) + when "3-plan" + plan_safe?(row) + else + unsafe(:stage_not_supported) + end + rescue StandardError => e + unsafe(:"error_#{e.class.name}") + end + + private + + def verify_marker(row, observed_marker_id, observed_reason, re_read:) + state_file = row_get(row, :state_file).to_s + if re_read && !state_file.empty? && File.file?(state_file) + marker = Hive::Markers.current(state_file) + name = marker.none? ? "none" : marker.name.to_s + attrs = marker.none? ? {} : marker.attrs + else + name = row_get(row, :marker).to_s.downcase + attrs = row_get(row, :marker_attrs) || {} + end + + if %w[execute_complete complete review_complete].include?(name) + return unsafe(:terminal_success) + end + unless name == "error" + return unsafe(:marker_changed) + end + + reason = attrs["reason"] || attrs[:reason] + marker_id = attrs["marker_id"] || attrs[:marker_id] + if reason.to_s != observed_reason.to_s + return unsafe(:marker_reason_changed) + end + if !observed_marker_id.to_s.empty? && marker_id.to_s != observed_marker_id.to_s + return unsafe(:marker_id_changed) + end + + Result.new(safe: true, rationale: "marker_ok") + end + + def execute_safe?(row) + worktree = row_get(row, :worktree_path).to_s + if worktree.empty? + # Try task reconstruction + folder = row_get(row, :folder).to_s + begin + task = Hive::Task.new(folder) + worktree = task.worktree_path.to_s + rescue StandardError + return unsafe(:worktree_unresolved) + end + end + + return unsafe(:worktree_missing) if worktree.empty? || !File.directory?(worktree) + return unsafe(:not_a_git_repo) unless File.directory?(File.join(worktree, ".git")) || + File.file?(File.join(worktree, ".git")) + + result = @runner.run( + [ "git", "-C", worktree, "status", "--porcelain" ], + timeout_sec: ProbeRunner::DEFAULT_STATUS_TIMEOUT_SEC + ) + return unsafe(result.timed_out ? :git_status_timeout : :git_status_failed) unless result.healthy? + return unsafe(:dirty_worktree) unless result.stdout.to_s.empty? + + Result.new(safe: true, rationale: "execute_clean") + rescue StandardError + unsafe(:execute_check_error) + end + + def brainstorm_safe?(row) + folder = row_get(row, :folder).to_s + path = File.join(folder, "brainstorm.md") + return Result.new(safe: true, rationale: "blank_artifact") unless File.file?(path) + + text = File.read(path, encoding: "UTF-8").scrub + return Result.new(safe: true, rationale: "blank_artifact") if blank_or_marker_only?(text) + + questions = Hive::BrainstormParser.parse_text(text) + if questions.any?(&:answered?) + return unsafe(:answered_questions) + end + # Malformed substantive content without parseable questions is unsafe + # if it looks like more than markers/blank. + if questions.empty? && text.strip.length > 0 && !blank_or_marker_only?(text) + # Parser returned nothing but file has prose — fail closed when it + # contains non-header substantive lines that aren't Q-structure. + return unsafe(:malformed_brainstorm) if substantive_non_qa?(text) + end + + Result.new(safe: true, rationale: "brainstorm_unanswered") + rescue StandardError + unsafe(:brainstorm_check_error) + end + + def plan_safe?(row) + folder = row_get(row, :folder).to_s + path = File.join(folder, "plan.md") + return Result.new(safe: true, rationale: "blank_artifact") unless File.file?(path) + + text = File.read(path, encoding: "UTF-8").scrub + # Strip only Hive-generated marker comments. Arbitrary HTML comments + # may be user feedback and therefore make replay unsafe. + cleaned = text.each_line.reject { |line| generated_marker_line?(line) }.join + return Result.new(safe: true, rationale: "blank_or_marker_only") if cleaned.strip.empty? + + unsafe(:plan_has_content) + rescue StandardError + unsafe(:plan_check_error) + end + + def blank_or_marker_only?(text) + text.each_line.all? do |line| + stripped = line.strip + stripped.empty? || generated_marker_line?(line) + end + end + + def substantive_non_qa?(text) + text.each_line.any? do |line| + stripped = line.strip + next false if stripped.empty? + next false if generated_marker_line?(line) + next false if stripped.match?(/\A\#{1,3}\s/) + + true + end + end + + def row_get(row, key) + if row.respond_to?(key) + row.public_send(key) + elsif row.is_a?(Hash) + row[key] || row[key.to_s] + end + end + + def generated_marker_line?(line) + line.to_s.match?(GENERATED_MARKER_LINE) + end + + def live_task_lock?(folder) + lock_path = File.join(folder, ".lock") + return false unless File.exist?(lock_path) + + !Hive::Lock.stale_lock?(lock_path) + rescue StandardError + true + end + + def unsafe(rationale) + Result.new(safe: false, rationale: rationale.to_s) + end + end + end + end +end diff --git a/lib/hive/daemon/auto_retry/state.rb b/lib/hive/daemon/auto_retry/state.rb new file mode 100644 index 00000000..eca8e788 --- /dev/null +++ b/lib/hive/daemon/auto_retry/state.rb @@ -0,0 +1,307 @@ +# frozen_string_literal: true + +require "json" +require "fileutils" +require "time" +require "hive/paths" +require "hive/daemon/auto_retry/policy" + +module Hive + module Daemon + module AutoRetry + # Durable retry ledger under state_home. Keyed by project identity, + # task id (slug fallback), stage, and marker reason — NOT marker_id. + # Fail-closed: corrupt/unreadable/newer-schema/unwritable state + # suspends automatic mutation rather than resetting budgets. + class State + SCHEMA_VERSION = 1 + STALE_TMP_SEC = 60 + FILENAME = "daemon_auto_retry.json" + + PHASES = %w[ + candidate clear_queued marker_cleared retry_queued retry_dispatched + failed succeeded aborted exhausted + ].freeze + + Error = Class.new(StandardError) + Suspended = Class.new(Error) + + def self.default_path + File.join(Hive::Paths.state_home, FILENAME) + end + + def initialize(path: self.class.default_path, logger: nil, clock: -> { Time.now }) + @path = path ? File.expand_path(path) : nil + @logger = logger + @clock = clock + @suspend_writes = false + @suspend_reason = nil + clean_orphaned_tmp_files! + end + + attr_reader :suspend_writes, :suspend_reason + + def suspended? + @suspend_writes + end + + # @return [Hash, nil] record or nil + def get(project:, task_key:, stage:, reason:) + all = load_all + all[key_for(project, task_key, stage, reason)] + end + + def find_by_recovery_id(recovery_id) + id = recovery_id.to_s + return nil if id.empty? + + matches = load_all.values.select do |record| + record.is_a?(Hash) && record["recovery_id"].to_s == id + end + raise Error, "duplicate_recovery_id" if matches.length > 1 + + matches.first + end + + def upsert!(project:, task_key:, stage:, reason:, **fields) + raise Suspended, @suspend_reason || "writes_suspended" if @suspend_writes + raise Suspended, "no_path" unless @path + + with_lock do + data = read_document + records = data.fetch("records") + k = key_for(project, task_key, stage, reason) + existing = records[k].is_a?(Hash) ? records[k] : default_record(project, task_key, stage, reason) + merged = existing.merge(stringify_fields(fields)) + merged["project"] = project.to_s + merged["task_key"] = task_key.to_s + merged["stage"] = stage.to_s + merged["reason"] = reason.to_s + merged["updated_at"] = @clock.call.utc.iso8601(6) + records[k] = merged + write_document!(data.merge("records" => records, "schema_version" => SCHEMA_VERSION)) + merged + end + end + + def mark_dispatched!(project:, task_key:, stage:, reason:, fingerprint:, recovery_id:) + raise Suspended, @suspend_reason || "writes_suspended" if @suspend_writes + + with_lock do + data = read_document + records = data.fetch("records") + k = key_for(project, task_key, stage, reason) + rec = records[k] + raise Error, "missing_retry_record" unless rec.is_a?(Hash) + existing_recovery_id = rec["recovery_id"].to_s + unless !existing_recovery_id.empty? && existing_recovery_id == recovery_id.to_s + raise Error, "recovery_id_mismatch" + end + if rec["phase"].to_s == "retry_dispatched" + return rec + end + unless %w[marker_cleared retry_queued].include?(rec["phase"].to_s) + raise Error, "invalid_dispatch_phase:#{rec['phase']}" + end + attempts = rec["attempts_dispatched"].to_i + 1 + raise Error, "attempt_budget_exhausted" if attempts > Policy::MAX_ATTEMPTS + rec = rec.merge( + "attempts_dispatched" => attempts, + "last_attempted_fingerprint" => fingerprint.to_s, + "phase" => "retry_dispatched", + "recovery_id" => recovery_id.to_s, + "updated_at" => @clock.call.utc.iso8601(6) + ) + if attempts >= Policy::MAX_ATTEMPTS + # Exhaustion is finalized when the next same-reason failure is + # observed; here we only count the dispatch. + end + records[k] = rec + write_document!(data.merge("records" => records, "schema_version" => SCHEMA_VERSION)) + rec + end + end + + def mark_exhausted!(project:, task_key:, stage:, reason:) + upsert!(project: project, task_key: task_key, stage: stage, reason: reason, + phase: "exhausted") + end + + def rearm_manual!(project:, task_key:, stage:, reason:) + raise Suspended, @suspend_reason || "writes_suspended" if @suspend_writes + + with_lock do + data = read_document + records = data.fetch("records") + k = key_for(project, task_key, stage, reason) + records[k] = default_record(project, task_key, stage, reason).merge( + "phase" => "candidate", + "manual_epoch_at" => @clock.call.utc.iso8601(6), + "updated_at" => @clock.call.utc.iso8601(6) + ) + write_document!(data.merge("records" => records, "schema_version" => SCHEMA_VERSION)) + records[k] + end + end + + def load_all + return {} if @suspend_writes + return {} unless @path && File.exist?(@path) + + read_document.fetch("records") + rescue Suspended + {} + end + + def key_for(project, task_key, stage, reason) + [ project, task_key, stage, reason ].map { |p| p.to_s.gsub("|", "_") }.join("|") + end + + def self.task_key(task_id: nil, slug: nil) + id = task_id.to_s.strip + return id unless id.empty? + + slug.to_s + end + + private + + def default_record(project, task_key, stage, reason) + { + "project" => project.to_s, + "task_key" => task_key.to_s, + "stage" => stage.to_s, + "reason" => reason.to_s, + "phase" => "candidate", + "attempts_dispatched" => 0, + "last_attempted_fingerprint" => nil, + "second_candidate_fingerprint" => nil, + "second_candidate_first_seen_at" => nil, + "eligible_at" => nil, + "last_probe_at" => nil, + "last_probe_fingerprint" => nil, + "last_probe_healthy" => nil, + "last_probe_rationale" => nil, + "last_probe_signal" => nil, + "marker_id" => nil, + "marker_signature" => nil, + "recovery_id" => nil, + "recovery_class" => nil, + "slug" => nil, + "folder" => nil, + "clear_argv" => nil, + "rerun_argv" => nil, + "clear_request_id" => nil, + "retry_request_id" => nil, + "last_negative_signature" => nil, + "last_negative_at" => nil, + "updated_at" => @clock.call.utc.iso8601(6) + } + end + + def stringify_fields(fields) + fields.each_with_object({}) do |(k, v), acc| + key = k.to_s + acc[key] = case v + when Time then v.utc.iso8601(6) + when Symbol then v.to_s + else v + end + end + end + + def read_document + return { "schema_version" => SCHEMA_VERSION, "records" => {} } unless @path && File.exist?(@path) + + parsed = JSON.parse(File.read(@path)) + suspend_and_raise!("root_not_hash") unless parsed.is_a?(Hash) + + version = parsed["schema_version"] + suspend_and_raise!("missing_schema_version") unless version.is_a?(Integer) + if version > SCHEMA_VERSION + @suspend_writes = true + @suspend_reason = "newer_schema" + raise Suspended, "newer_schema" + end + records = parsed["records"] + suspend_and_raise!("records_not_hash") unless records.is_a?(Hash) + suspend_and_raise!("record_not_hash") unless records.values.all? { |record| record.is_a?(Hash) } + parsed + rescue JSON::ParserError, TypeError, SystemCallError, IOError => e + suspend_and_raise!("corrupt:#{e.class}") + end + + def write_document!(doc) + raise Suspended, "no_path" unless @path + + dir = File.dirname(@path) + FileUtils.mkdir_p(dir) + tmp = "#{@path}.tmp.#{Process.pid}.#{Thread.current.object_id}" + File.open(tmp, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |f| + f.write(JSON.pretty_generate(doc)) + f.flush + f.fsync + end + File.rename(tmp, @path) + fsync_dir(dir) + rescue SystemCallError, IOError => e + suspend_and_raise!("write_error:#{e.class}") + ensure + FileUtils.rm_f(tmp) if defined?(tmp) && tmp && File.exist?(tmp) + end + + def fsync_dir(dir) + d = Dir.open(dir) + d.fsync if d.respond_to?(:fsync) + rescue StandardError + nil + ensure + d&.close + end + + def with_lock + return yield unless @path + + lock_path = "#{@path}.lock" + FileUtils.mkdir_p(File.dirname(lock_path)) + File.open(lock_path, File::RDWR | File::CREAT, 0o644) do |lf| + lf.flock(File::LOCK_EX) + yield + end + rescue SystemCallError => e + @logger&.event(:daemon_auto_retry_lock_error, + path: @path, error_class: e.class.name, message: e.message) + raise Suspended, "lock_error:#{e.class}" + end + + def suspend!(reason) + @suspend_writes = true + @suspend_reason = reason + @logger&.event(:daemon_auto_retry_state_suspended, path: @path, reason: reason) + {} + end + + def suspend_and_raise!(reason) + suspend!(reason) + raise Suspended, reason + end + + def clean_orphaned_tmp_files! + return unless @path + + dir = File.dirname(@path) + return unless File.directory?(dir) + + Dir.glob(File.join(dir, "#{File.basename(@path)}.tmp.*")).each do |tmp| + next unless File.file?(tmp) + next if (Time.now - File.mtime(tmp)) < STALE_TMP_SEC + + FileUtils.rm_f(tmp) + rescue StandardError + nil + end + end + end + end + end +end diff --git a/lib/hive/daemon/dispatch_request_queue.rb b/lib/hive/daemon/dispatch_request_queue.rb index 00d7aacc..9e52a030 100644 --- a/lib/hive/daemon/dispatch_request_queue.rb +++ b/lib/hive/daemon/dispatch_request_queue.rb @@ -2,6 +2,7 @@ require "json" require "fileutils" require "securerandom" require "time" +require "digest" require "hive/paths" require "hive/daemon/queue_directory" @@ -30,7 +31,7 @@ module Hive Request = Struct.new( :request_id, :created_at, :project, :slug, :argv, :requestor, - :chat_id, :update_id, :trigger, :path, + :chat_id, :update_id, :trigger, :path, :recovery_id, :recovery_step, keyword_init: true ) @@ -48,14 +49,35 @@ module Hive SecureRandom.hex(8) end + ALLOWED_REQUESTORS = %w[bot healer auto_retry].freeze + def write_request!(project:, slug:, argv:, requestor: "bot", chat_id: nil, update_id: nil, trigger: nil, request_id: generate_request_id, - state_home: Hive::Paths.state_home, now: Time.now) + state_home: Hive::Paths.state_home, now: Time.now, + recovery_id: nil, recovery_step: nil) unless valid_argv?(argv) raise ArgumentError, "argv #{argv.inspect} is not allowlisted for dispatch requests" end raise ArgumentError, "project is required for dispatch requests" if project.to_s.empty? raise ArgumentError, "slug is required for dispatch requests" if slug.to_s.empty? + unless ALLOWED_REQUESTORS.include?(requestor.to_s) + raise ArgumentError, "requestor #{requestor.inspect} is not allowlisted" + end + + unless recovery_id.to_s.empty? + existing = metadata(request_id, state_home: state_home) + if existing + coherent = existing[:project].to_s == project.to_s && + existing[:slug].to_s == slug.to_s && + existing[:requestor].to_s == requestor.to_s && + existing[:recovery_id].to_s == recovery_id.to_s && + existing[:recovery_step].to_s == recovery_step.to_s && + Array(existing[:argv]) == Array(argv) + return request_id.to_s if coherent + + raise ArgumentError, "request_id #{request_id.inspect} collides with different recovery metadata" + end + end created_at = now.utc payload = { @@ -71,6 +93,10 @@ module Hive "update_id" => update_id, "trigger" => trigger.to_s } + payload["recovery_id"] = recovery_id.to_s if recovery_id && !recovery_id.to_s.empty? + if recovery_step && !recovery_step.to_s.empty? + payload["recovery_step"] = recovery_step.to_s + end dir = directory(state_home: state_home) filename = filename_for(created_at: created_at, request_id: request_id) @@ -82,6 +108,7 @@ module Hive f.fsync end File.rename(tmp_path, final_path) + fsync_directory(dir) request_id.to_s ensure FileUtils.rm_f(tmp_path) if defined?(tmp_path) && tmp_path && File.exist?(tmp_path) @@ -272,7 +299,9 @@ module Hive return { chat_id: data["chat_id"], update_id: data["update_id"], project: data["project"], slug: data["slug"], - requestor: data["requestor"] + requestor: data["requestor"], + recovery_id: data["recovery_id"], recovery_step: data["recovery_step"], + argv: data["argv"] } end nil @@ -280,7 +309,12 @@ module Hive nil end - def write_sequence!(request_id, remaining_argvs:, state_home: Hive::Paths.state_home) + def request_exists?(request_id, state_home: Hive::Paths.state_home) + !metadata(request_id, state_home: state_home).nil? + end + + def write_sequence!(request_id, remaining_argvs:, state_home: Hive::Paths.state_home, + recovery_id: nil) remaining = Array(remaining_argvs) return discard_sequence(request_id, state_home: state_home) if remaining.empty? @@ -290,12 +324,15 @@ module Hive path = sequence_path(directory(state_home: state_home), request_id) tmp_path = "#{path}.tmp.#{Process.pid}.#{Thread.current.object_id}" + payload = { "request_id" => request_id.to_s, "remaining_argvs" => remaining } + payload["recovery_id"] = recovery_id.to_s if recovery_id && !recovery_id.to_s.empty? File.open(tmp_path, File::WRONLY | File::CREAT | File::TRUNC, 0o600) do |f| - f.write(JSON.generate("request_id" => request_id.to_s, "remaining_argvs" => remaining)) + f.write(JSON.generate(payload)) f.flush f.fsync end File.rename(tmp_path, path) + fsync_directory(File.dirname(path)) true ensure FileUtils.rm_f(tmp_path) if defined?(tmp_path) && tmp_path && File.exist?(tmp_path) @@ -314,28 +351,41 @@ module Hive def promote_sequence(request_id, project:, slug:, requestor: "bot", chat_id: nil, update_id: nil, trigger: "sequence_continuation", - state_home: Hive::Paths.state_home, now: Time.now) + state_home: Hive::Paths.state_home, now: Time.now, + recovery_id: nil) dir = directory(state_home: state_home) path = sequence_path(dir, request_id) - sequence = read_sequence(path) + sequence_doc = read_sequence_doc(path) + sequence = Array(sequence_doc["remaining_argvs"]) + recovery_id = recovery_id || sequence_doc["recovery_id"] return nil if sequence.empty? next_argv = sequence.shift + next_request_id = if recovery_id && !recovery_id.to_s.empty? + ::Digest::SHA256.hexdigest("#{recovery_id}:rerun")[0, 16] + else + generate_request_id + end next_request_id = write_request!( project: project, slug: slug, argv: next_argv, requestor: requestor, chat_id: chat_id, update_id: update_id, trigger: trigger, - state_home: state_home, now: now + state_home: state_home, now: now, + recovery_id: recovery_id, + recovery_step: recovery_id ? "rerun" : nil, + request_id: next_request_id ) if sequence.empty? FileUtils.rm_f(path) else - write_sequence!(next_request_id, remaining_argvs: sequence, state_home: state_home) + write_sequence!(next_request_id, remaining_argvs: sequence, state_home: state_home, + recovery_id: recovery_id) FileUtils.rm_f(path) end Request.new( request_id: next_request_id, created_at: now.utc, project: project.to_s, slug: slug.to_s, argv: next_argv, requestor: requestor.to_s, - chat_id: chat_id, update_id: update_id, trigger: trigger.to_s, path: nil + chat_id: chat_id, update_id: update_id, trigger: trigger.to_s, path: nil, + recovery_id: recovery_id, recovery_step: recovery_id ? "rerun" : nil ) end @@ -407,14 +457,20 @@ module Hive end def read_sequence(path) + Array(read_sequence_doc(path)["remaining_argvs"]) + end + + def read_sequence_doc(path) data = JSON.parse(File.read(path)) - remaining = data.is_a?(Hash) ? data["remaining_argvs"] : nil - return [] unless remaining.is_a?(Array) - return [] unless remaining.all? { |argv| valid_argv?(argv) } + return {} unless data.is_a?(Hash) + + remaining = data["remaining_argvs"] + return {} unless remaining.is_a?(Array) + return {} unless remaining.all? { |argv| valid_argv?(argv) } - remaining + data rescue Errno::ENOENT, JSON::ParserError, IOError - [] + {} end # Persist directory entries (renames/unlinks) so they survive an @@ -508,7 +564,9 @@ module Hive chat_id: data["chat_id"], update_id: data["update_id"], trigger: data["trigger"].to_s, - path: path + path: path, + recovery_id: data["recovery_id"], + recovery_step: data["recovery_step"] ) end diff --git a/lib/hive/daemon/dispatcher.rb b/lib/hive/daemon/dispatcher.rb index 1f272c69..ade67742 100644 --- a/lib/hive/daemon/dispatcher.rb +++ b/lib/hive/daemon/dispatcher.rb @@ -21,8 +21,11 @@ require "hive/daemon/digest_scheduler" require "hive/daemon/answer_digest_scheduler" require "hive/daemon/patrol_scheduler" require "hive/daemon/pr_merge_watcher" +require "hive/daemon/auto_retry/coordinator" +require "hive/daemon/auto_retry/state" require "hive/lock" require "hive/paths" +require "hive/runtime_identity" require "hive/update_check" require "hive/update_check/state" require "hive/install_channel" @@ -159,6 +162,22 @@ module Hive # see the next warning the next time it goes red, but we don't # actively re-emit on every tick. Issue #95. @legacy_layout_logged = {} + # Health-gated auto-retry coordinator (v1). Own subsystem; not part + # of StaleAgentHealer. Default-on via daemon.auto_retry.enabled. + # Constructed after @enabled_cache / @legacy_layout_projects so the + # closures and tick-time legacy pointer stay valid. + auto_retry_cfg = @daemon_cfg["auto_retry"].is_a?(Hash) ? @daemon_cfg["auto_retry"] : {} + @auto_retry_enabled = auto_retry_cfg.fetch("enabled", true) != false + @auto_retry = AutoRetry::Coordinator.new( + logger: @logger, + controller: @controller, + state: AutoRetry::State.new(logger: @logger), + enabled: @auto_retry_enabled, + dry_run: @dry_run, + config_for_project: ->(name) { load_project_config_for_auto_retry(name) }, + project_enabled: ->(name) { project_enabled?(name) }, + legacy_projects: @legacy_layout_projects + ) # Test-injectable state homes for the dispatch-request and # dispatch-result queues. Production passes nil so both resolve # `Hive::Paths.state_home`; unit tests inject a sandbox. The result @@ -314,6 +333,19 @@ module Hive last_error: drop[:last_error]) end + # 3a. Health-gated auto-retry coordinator. Runs AFTER merge-watcher + # (finalize merge recovery keeps precedence) and BEFORE dispatch- + # request processing so a newly queued clear can pass normal queue + # gates in the same tick. + begin + @auto_retry.instance_variable_set(:@legacy_projects, @legacy_layout_projects) + @auto_retry.tick(result.rows, now: now) + rescue StandardError => e + @logger.event(:fatal, + message: "auto_retry coordinator raised: #{e.class}: #{e.message}", + keeping_previous: true) + end + # 3b. Dispatch-request queue (plan 2026-05-28-002). Process # bot-written request files BEFORE the per-row scan so a slug # whose request just spawned is already in-flight in the @@ -477,12 +509,90 @@ module Hive # as a cheap drift signal — if the on-disk file's digest no longer # matches what we captured at startup, the loaded code is stale. # Returns nil on any failure; a nil baseline disables drift checks - # so a transient read failure never re-execs. + # so a transient read failure never re-execs. Factored through + # RuntimeIdentity so re-exec detection and Claude health validation + # cannot disagree. def compute_code_fingerprint - path = Hive::Schemas.method(:schema_path).source_location.first - ::Digest::SHA256.file(path).hexdigest + Hive::RuntimeIdentity.code_fingerprint + end + + def load_project_config_for_auto_retry(project_name) + entry = Hive::Config.find_project(project_name) + return {} unless entry + + Hive::Config.load(entry["path"]) rescue StandardError - nil + {} + end + + def auto_retry_rerun_request?(req) + argv = Array(req.argv) + return false if argv[1].to_s == "markers" + + step = req.respond_to?(:recovery_step) ? req.recovery_step.to_s : "" + step == "rerun" + end + + def note_auto_retry_dispatch!(req) + recovery_id = req.respond_to?(:recovery_id) ? req.recovery_id : nil + recovery_id = recovery_id.to_s + return false if recovery_id.empty? + + state = @auto_retry.instance_variable_get(:@state) + match = state.find_by_recovery_id(recovery_id) + return false unless coherent_auto_retry_match?(req, match, expected_step: "rerun") + + @auto_retry.note_retry_dispatched!( + recovery_id: recovery_id, + project: req.project, + slug: req.slug, + stage: match["stage"], + reason: match["reason"] + ) + rescue Hive::Daemon::AutoRetry::State::Error => e + @logger.event(:auto_retry_state_error, message: e.message, + recovery_id: recovery_id) + false + end + + def auto_retry_request_authorized?(req) + return false unless @auto_retry_enabled + + recovery_id = req.respond_to?(:recovery_id) ? req.recovery_id.to_s : "" + step = req.respond_to?(:recovery_step) ? req.recovery_step.to_s : "" + return false if recovery_id.empty? || !%w[clear rerun].include?(step) + + state = @auto_retry.instance_variable_get(:@state) + match = state.find_by_recovery_id(recovery_id) + return false unless coherent_auto_retry_match?(req, match, expected_step: step) + + # marker_cleared is accepted for clear only to close the write-ahead + # crash window: the ledger may advance immediately before the marker + # rewrite, and reconciliation must be able to repeat that same clear. + allowed_phases = step == "clear" ? %w[clear_queued marker_cleared] : %w[marker_cleared retry_queued retry_dispatched] + allowed_phases.include?(match["phase"].to_s) + rescue Hive::Daemon::AutoRetry::State::Error => e + @logger.event(:auto_retry_state_error, message: e.message, + recovery_id: recovery_id) + false + end + + def coherent_auto_retry_match?(req, match, expected_step:) + return false unless match.is_a?(Hash) + return false if match["stage"].to_s.empty? || match["reason"].to_s.empty? + return false if match["recovery_id"].to_s.empty? + return false unless match["recovery_id"].to_s == req.recovery_id.to_s + return false unless match["project"].to_s == req.project.to_s + return false unless match["slug"].to_s == req.slug.to_s + + return false unless req.recovery_step.to_s == expected_step.to_s + + request_field = expected_step.to_s == "clear" ? "clear_request_id" : "retry_request_id" + argv_field = expected_step.to_s == "clear" ? "clear_argv" : "rerun_argv" + return false if match[request_field].to_s.empty? || Array(match[argv_field]).empty? + return false unless match[request_field].to_s == req.request_id.to_s + + Array(match[argv_field]) == Array(req.argv) end # True iff a baseline fingerprint exists, a fresh fingerprint can @@ -1315,6 +1425,12 @@ module Hive return end + if req.requestor.to_s == "auto_retry" && !auto_retry_request_authorized?(req) + reason = @auto_retry_enabled ? "auto_retry_authorization_failed" : "auto_retry_disabled" + reject_request(req, reason: reason) + return + end + if Hive::Daemon::DispatchRequestQueue.expired?(req, now: now) expire_request(req) return @@ -1366,7 +1482,7 @@ module Hive # Build a command string from the validated argv and spawn it # through the same `dispatch_command` path auto-advance uses. # - # C3: immediately after the spawn we CLAIM the request file — + # C3: immediately before the spawn we CLAIM the request file — # rename `.json` → `.json.claimed` and stamp the child's # pid + process_start_time. The claimed file is invisible to # `pending`, so a later tick never re-observes (or re-dispatches) @@ -1374,9 +1490,24 @@ module Hive # `recover_dispatch_claims` cleans up at next start instead of # re-running the work. The claimed file is unlinked on reap. def dispatch_request!(req, now:) + preclaim_dispatch_request(req, now: now) + + # Auto-retry attempt accounting: count only when the same-stage + # rerun is authorized for dispatch (not on marker clear). + if req.requestor.to_s == "auto_retry" && auto_retry_rerun_request?(req) + unless note_auto_retry_dispatch!(req) + @logger.event(:dispatch_request_blocked, + request_id: req.request_id, project: req.project, + slug: req.slug, reason: "auto_retry_state_write_failed") + Hive::Daemon::DispatchRequestQueue.release_claim( + req.request_id, state_home: dispatch_request_state_home + ) + return + end + end + command = Shellwords.join(req.argv) state_file_path = resolve_request_state_file_path(req) - preclaim_dispatch_request(req, now: now) pid = dispatch_command( command, project: req.project, slug: req.slug, @@ -1842,6 +1973,10 @@ module Hive logger: @logger, dry_run: @dry_run ) + # Toggle auto-retry kill switch without reconstructing durable state. + auto_retry_cfg = @daemon_cfg["auto_retry"].is_a?(Hash) ? @daemon_cfg["auto_retry"] : {} + @auto_retry_enabled = auto_retry_cfg.fetch("enabled", true) != false + @auto_retry&.reconfigure!(enabled: @auto_retry_enabled) @enabled_cache.clear @logger.event(:config_reloaded) rescue Hive::ConfigError => e diff --git a/lib/hive/daemon/logger.rb b/lib/hive/daemon/logger.rb index 6af59dc9..04c8d97c 100644 --- a/lib/hive/daemon/logger.rb +++ b/lib/hive/daemon/logger.rb @@ -76,6 +76,17 @@ module Hive digest_state_unreadable answer_digest_failure_backoff answer_digest_state_unreadable + auto_retry_decision + auto_retry_error + auto_retry_state_error + auto_retry_state_suspended + auto_retry_queue_error + auto_retry_sequence_error + auto_retry_audit_error + auto_retry_reconcile + daemon_auto_retry_newer_schema_suspended + daemon_auto_retry_state_suspended + daemon_auto_retry_lock_error fatal ].freeze diff --git a/lib/hive/daemon/status_consumer.rb b/lib/hive/daemon/status_consumer.rb index b483ad0a..2af9aec3 100644 --- a/lib/hive/daemon/status_consumer.rb +++ b/lib/hive/daemon/status_consumer.rb @@ -19,7 +19,7 @@ module Hive Row = Struct.new(:project, :slug, :stage, :workflow, :marker, :marker_attrs, :folder, :state_file, :state_file_mtime, :action, :suggested_command, :claude_pid_alive, :live_task_lock, :diagnostic, :depends_on, :blocked_by, - :dependency_stage, :blocked, + :dependency_stage, :blocked, :id, :worktree_path, keyword_init: true) # Aggregated per-project legacy-layout signal lifted out of each # project payload's `legacy_stage_dirs` array. The dispatcher uses @@ -196,7 +196,9 @@ module Hive depends_on: task["depends_on"], blocked_by: task["blocked_by"], dependency_stage: task["dependency_stage"], - blocked: task["blocked"] == true + blocked: task["blocked"] == true, + id: task["id"], + worktree_path: task["worktree_path"] ) end end diff --git a/lib/hive/events.rb b/lib/hive/events.rb index f8bca816..58883019 100644 --- a/lib/hive/events.rb +++ b/lib/hive/events.rb @@ -15,6 +15,7 @@ module Hive round_complete clean_exit_auto_committed claude_completion_fallback + auto_retry_decision ].freeze STATUS_TAIL_LINES = 20 @@ -46,7 +47,12 @@ module Hive # appenders via the inode lock; we cap message size (see # MAX_MESSAGE_BYTES) so the full line stays small and well-defined. # status.md is derived state and is rewritten with atomic rename. - def emit(task_folder:, slug:, stage:, event_type:, agent: nil, message: nil) + # Optional `details` is a bounded Hash of structured audit fields for + # auto-retry (and future) events. Existing base keys and the single- + # syswrite append contract are preserved; old readers ignore unknown keys. + MAX_DETAILS_BYTES = 2_048 + + def emit(task_folder:, slug:, stage:, event_type:, agent: nil, message: nil, details: nil) event_type = event_type.to_sym unless EVENT_TYPES.include?(event_type) raise ArgumentError, "unknown event_type #{event_type.inspect}; valid: #{EVENT_TYPES.inspect}" @@ -60,6 +66,9 @@ module Hive "event_type" => event_type.to_s, "message" => message.nil? ? nil : truncate_message(message.to_s) } + if details.is_a?(Hash) && !details.empty? + record["details"] = bound_details(details) + end FileUtils.mkdir_p(task_folder) events_path = File.join(task_folder, "events.jsonl") @@ -77,6 +86,43 @@ module Hive nil end + def bound_details(details) + # JSON-serialize and truncate so the full line stays under the append budget. + cleaned = details.each_with_object({}) do |(k, v), acc| + acc[k.to_s] = case v + when String, Numeric, TrueClass, FalseClass, NilClass then v + when Symbol then v.to_s + when Time then v.utc.iso8601 + when Hash, Array then v + else v.to_s + end + end + raw = JSON.generate(cleaned) + return cleaned if raw.bytesize <= MAX_DETAILS_BYTES + + bounded_preview(raw) + rescue StandardError + { "error" => "details_unserializable" } + end + + def bounded_preview(raw) + low = 0 + high = [ raw.bytesize, MAX_DETAILS_BYTES ].min + best = { "truncated" => true, "preview" => "" } + while low <= high + mid = (low + high) / 2 + preview = raw.byteslice(0, mid).to_s.scrub("") + candidate = { "truncated" => true, "preview" => preview } + if JSON.generate(candidate).bytesize <= MAX_DETAILS_BYTES + best = candidate + low = mid + 1 + else + high = mid - 1 + end + end + best + end + def truncate_message(message) return message if message.bytesize <= MAX_MESSAGE_BYTES diff --git a/lib/hive/recovery_sequence.rb b/lib/hive/recovery_sequence.rb new file mode 100644 index 00000000..cf3d9da9 --- /dev/null +++ b/lib/hive/recovery_sequence.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +require "hive/workflows" +require "hive/workflows/project" +require "hive/config" + +module Hive + # Daemon-neutral pure argv builder for marker-clear + same-stage retry. + # Extracted so bot recovery and daemon auto-retry emit byte-equivalent + # workflow verbs and --from/--stage scoping. + module RecoverySequence + module_function + + # @return [Array>] ordered argvs: optional clear, then retry + def commands(project:, slug:, stage:, marker:, match_attr: nil, workflow: nil) + verb = retry_verb_for_stage(stage, workflow: workflow, project: project) + return [] unless verb + + commands = [] + marker_name = marker.to_s + unless marker_name.casecmp("none").zero? || marker_name.casecmp("agent_working").zero? + clear_argv = [ "hive", "markers", "clear", slug, "--name", marker_name.upcase, + "--project", project ] + clear_argv += [ "--match-attr", match_attr ] if match_attr.to_s.include?("=") + clear_argv << "--json" + commands << clear_argv + end + stage_flag = verb == "run" ? "--stage" : "--from" + commands << [ "hive", verb, slug, stage_flag, stage, "--project", project, "--json" ] + commands + end + + def retry_verb_for_stage(stage, workflow: nil, project: nil) + stage = stage.to_s + unless Hive::Workflows.coding_id?(workflow) + return nil if generic_terminal_stage?(stage, workflow, project: project) + return nil if generic_non_agent_stage?(stage, workflow, project: project) + + return "run" + end + return nil if stage == "9-done" # coding-scoped: coding workflow terminal stage + + # coding-scoped (block): legacy coding stage aliases + Hive::Workflows.verb_arriving_at(stage) || { + "5-review" => "review", + "6-pr" => "pr" + }[stage] + end + + def match_attr_for_error(attrs) + require "hive/markers" + Hive::Markers.error_recovery_match_attr(attrs || {}) + end + + def generic_terminal_stage?(stage, workflow, project: nil) + descriptor = resolve_descriptor(workflow, project: project) + return false unless descriptor + + last = descriptor.stages.last + !last.nil? && last.dir == stage + end + private_class_method :generic_terminal_stage? + + def generic_non_agent_stage?(stage, workflow, project: nil) + descriptor = resolve_descriptor(workflow, project: project) + return false unless descriptor + + found = descriptor.stage_for_dir(stage) + !found.nil? && found.kind != :agent + end + private_class_method :generic_non_agent_stage? + + def resolve_descriptor(workflow, project: nil) + Hive::Workflows::Project.synchronize do + load_project_overlay(project) + Hive::Workflows::Registry.fetch(workflow.to_s.to_sym) + end + rescue Hive::Workflows::UnknownWorkflow + nil + end + private_class_method :resolve_descriptor + + def load_project_overlay(project_name) + return if project_name.nil? || project_name.to_s.empty? + + match = Hive::Config.registered_projects.find { |p| p["name"] == project_name.to_s } + Hive::Workflows::Project.load!(match["path"]) if match + end + private_class_method :load_project_overlay + end +end diff --git a/lib/hive/runtime_identity.rb b/lib/hive/runtime_identity.rb new file mode 100644 index 00000000..b41416f5 --- /dev/null +++ b/lib/hive/runtime_identity.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "digest" +require "hive" + +module Hive + # Canonical identity of the running hive process / CLI install. + # Shared by daemon re-exec drift detection and auto-retry Claude health + # probes so the two surfaces cannot disagree about "which code is live". + module RuntimeIdentity + module_function + + # SHA-256 of the file that defines SCHEMA_VERSIONS (lib/hive.rb source + # location for Schemas helpers, matching Dispatcher#compute_code_fingerprint). + def code_fingerprint + path = Hive::Schemas.method(:schema_path).source_location.first + ::Digest::SHA256.file(path).hexdigest + rescue StandardError + nil + end + + def version + Hive::VERSION.to_s + end + + def binary_path(program_name: $PROGRAM_NAME, env: ENV) + require "hive/invoked_binary" + path = Hive::InvokedBinary.path(program_name: program_name, env: env) + return nil if path.nil? + + File.realpath(path) + rescue SystemCallError + path + end + + # Canonical, non-secret identity hash used as a health-fingerprint input. + def snapshot(program_name: $PROGRAM_NAME, env: ENV) + { + "version" => version, + "code_fingerprint" => code_fingerprint, + "binary_path" => binary_path(program_name: program_name, env: env) + } + end + + def matches_cli?(cli_version:, cli_fingerprint:, cli_binary_path:, + program_name: $PROGRAM_NAME, env: ENV) + snap = snapshot(program_name: program_name, env: env) + return false if snap["version"].to_s.empty? || cli_version.to_s.empty? + return false unless snap["version"].to_s == cli_version.to_s + return false if snap["code_fingerprint"].nil? || cli_fingerprint.to_s.empty? + return false unless snap["code_fingerprint"].to_s == cli_fingerprint.to_s + + daemon_bin = snap["binary_path"].to_s + cli_bin = cli_binary_path.to_s + return false if daemon_bin.empty? || cli_bin.empty? + + begin + File.realpath(daemon_bin) == File.realpath(cli_bin) + rescue SystemCallError + daemon_bin == cli_bin + end + end + end +end diff --git a/schemas/hive-dispatch-request.v2.json b/schemas/hive-dispatch-request.v2.json index a720ff0a..ace01f3a 100644 --- a/schemas/hive-dispatch-request.v2.json +++ b/schemas/hive-dispatch-request.v2.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/ivankuznetsov/hive/blob/main/schemas/hive-dispatch-request.v2.json", "title": "hive dispatch request (v2)", - "description": "One JSON file under /dispatch_requests/, atomic-written by a non-daemon producer and consumed by the daemon dispatcher. v2 is strict-version-matched — any schema_version != 2 is rejected with reason=unknown_schema_version and the file removed. v2 over v1: the requestor enum gains 'healer' (the daemon's stale-agent healer re-enqueues plan reruns through the same queue). Registered producers: 'bot' (the Telegram bot AND the web dispatcher, which rides Hive::Bot::DispatchRequestWriter) and 'healer'. The single-dispatcher invariant still holds: producers only enqueue; the daemon is the sole executor, and allowed verbs are the closed set in argv[1] below. Forward-compat requires a coordinated producer→daemon upgrade: bump SCHEMA_VERSION and add a v3 schema before any producer emits it.", + "description": "One JSON file under /dispatch_requests/, atomic-written by a producer and consumed by the daemon dispatcher. v2 is strict-version-matched — any schema_version != 2 is rejected with reason=unknown_schema_version and the file removed. Registered producers are 'bot' (Telegram and web), 'healer' (stale-agent plan requeue), and 'auto_retry' (health-gated clear/rerun with durable recovery metadata). The single-dispatcher invariant still holds: producers only enqueue; the daemon is the sole executor, and allowed verbs are the closed set in argv[1] below. Forward-compat requires a coordinated producer→daemon upgrade: bump SCHEMA_VERSION and add a v3 schema before any producer emits it.", "type": "object", "additionalProperties": false, "required": [ @@ -56,9 +56,29 @@ "type": "string", "enum": [ "bot", - "healer" + "healer", + "auto_retry" ], - "description": "Identity of the producer. 'bot' covers the Telegram bot and the web dispatcher (both write via Hive::Bot::DispatchRequestWriter); 'healer' is the daemon's stale-agent healer. Note: the daemon does NOT verify this field against a process credential — the queue dir's filesystem permissions (mode 0700) are the de-facto auth boundary." + "description": "Identity of the producer. 'bot' covers the Telegram bot and the web dispatcher (both write via Hive::Bot::DispatchRequestWriter); 'healer' is the daemon's stale-agent healer; 'auto_retry' is the health-gated daemon auto-retry coordinator. Note: the daemon does NOT verify this field against a process credential — the queue dir's filesystem permissions (mode 0700) are the de-facto auth boundary." + }, + "recovery_id": { + "type": [ + "string", + "null" + ], + "description": "Optional auto-retry recovery episode id. When present, ties this request to a durable auto-retry ledger entry so automatic clears cannot rearm their own attempt budget." + }, + "recovery_step": { + "type": [ + "string", + "null" + ], + "enum": [ + "clear", + "rerun", + null + ], + "description": "Optional auto-retry step within a recovery episode." }, "chat_id": { "type": [ diff --git a/templates/hive_config.yml.erb b/templates/hive_config.yml.erb index 99d05dc9..c5ca3479 100644 --- a/templates/hive_config.yml.erb +++ b/templates/hive_config.yml.erb @@ -46,3 +46,13 @@ registered_projects: <%= registered_projects.empty? ? "[]" : "" %> # answer_digest: # enabled: false # hour: 9 + +# Health-gated daemon auto-retry (v1). Defaults ON. When true, the daemon may +# clear and re-run a narrow allowlist of terminal ERROR markers after proving +# agent health and replay safety. Set enabled: false to park all such markers +# for manual recovery without affecting StaleAgentHealer or per-project +# daemon.enabled. +# +# daemon: +# auto_retry: +# enabled: true diff --git a/test/fixtures/auto_retry/near_miss/claude-quota.log b/test/fixtures/auto_retry/near_miss/claude-quota.log new file mode 100644 index 00000000..bf592d4a --- /dev/null +++ b/test/fixtures/auto_retry/near_miss/claude-quota.log @@ -0,0 +1 @@ +2026-05-20T09:05:00Z claude interactive prompt did not become ready; limits reached for claude: Stop and wait for limit to reset diff --git a/test/fixtures/auto_retry/near_miss/claude-session-exists.log b/test/fixtures/auto_retry/near_miss/claude-session-exists.log new file mode 100644 index 00000000..2e1a3b54 --- /dev/null +++ b/test/fixtures/auto_retry/near_miss/claude-session-exists.log @@ -0,0 +1 @@ +2026-05-20T09:00:00Z tmux session hive-foo already exists; attach with `tmux attach -t hive-foo` or kill it first diff --git a/test/fixtures/auto_retry/near_miss/generic-exit-1.log b/test/fixtures/auto_retry/near_miss/generic-exit-1.log new file mode 100644 index 00000000..921dfa91 --- /dev/null +++ b/test/fixtures/auto_retry/near_miss/generic-exit-1.log @@ -0,0 +1,2 @@ +2026-05-12T12:00:00Z agent exited with exit_code=1 +2026-05-12T12:00:00Z implementer failed for unknown reason diff --git a/test/fixtures/auto_retry/near_miss/unrelated-401.log b/test/fixtures/auto_retry/near_miss/unrelated-401.log new file mode 100644 index 00000000..b29852b1 --- /dev/null +++ b/test/fixtures/auto_retry/near_miss/unrelated-401.log @@ -0,0 +1,2 @@ +2026-05-12T12:30:00Z github api returned 401 Unauthorized for token scopes +2026-05-12T12:30:00Z implementer failed while calling gh diff --git a/test/fixtures/auto_retry/task-287/launcher.log b/test/fixtures/auto_retry/task-287/launcher.log new file mode 100644 index 00000000..45b9a159 --- /dev/null +++ b/test/fixtures/auto_retry/task-287/launcher.log @@ -0,0 +1,2 @@ +2026-05-20T08:15:00Z launching claude via interactive_claude_wrapper.sh +2026-05-20T08:15:12Z claude interactive prompt did not become ready in tmux session hive-task-287; last pane tail: "..." diff --git a/test/fixtures/auto_retry/task-58/agent.log b/test/fixtures/auto_retry/task-58/agent.log new file mode 100644 index 00000000..e82e5c31 --- /dev/null +++ b/test/fixtures/auto_retry/task-58/agent.log @@ -0,0 +1,3 @@ +2026-05-12T10:00:01Z codex exec starting +2026-05-12T10:00:02Z stream error: unexpected status 401 Unauthorized: missing bearer authentication; check login status +2026-05-12T10:00:02Z implementer exited with status error diff --git a/test/fixtures/auto_retry/task-58/basic-auth.log b/test/fixtures/auto_retry/task-58/basic-auth.log new file mode 100644 index 00000000..63d55c84 --- /dev/null +++ b/test/fixtures/auto_retry/task-58/basic-auth.log @@ -0,0 +1,3 @@ +2026-05-12T11:00:01Z codex exec starting +2026-05-12T11:00:02Z ERROR: HTTP 401 Unauthorized — basic authentication required / missing basic-auth credentials +2026-05-12T11:00:02Z implementer failed diff --git a/test/integration/daemon_auto_retry_test.rb b/test/integration/daemon_auto_retry_test.rb new file mode 100644 index 00000000..e3831f3e --- /dev/null +++ b/test/integration/daemon_auto_retry_test.rb @@ -0,0 +1,277 @@ +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" +require "fileutils" +require "json" +require "hive/daemon/auto_retry/coordinator" +require "hive/daemon/auto_retry/state" +require "hive/daemon/auto_retry/health" +require "hive/daemon/auto_retry/classifier" +require "hive/daemon/status_consumer" +require "hive/daemon/dispatch_request_queue" +require "hive/config" + +# Hermetic acceptance paths for v1 health-gated auto-retry (task-58 / +# task-287 style fixtures with fake health). No live provider calls. +class DaemonAutoRetryIntegrationTest < Minitest::Test + Row = Hive::Daemon::StatusConsumer::Row + + class FakeLogger + attr_reader :events + def initialize = @events = [] + def event(name, **attrs) = @events << [ name, attrs ] + end + + class FakeController + def running_task?(**) = false + end + + class RecordingQueue + attr_reader :requests, :sequences + def initialize + @requests = [] + @sequences = {} + end + def pending = [] + def write_request!(**kwargs) + @requests << kwargs + kwargs[:request_id] || "req-#{@requests.size}" + end + def write_sequence!(id, remaining_argvs:, state_home: nil, recovery_id: nil) + @sequences[id] = { argvs: remaining_argvs, recovery_id: recovery_id } + true + end + def remove_if_unclaimed(*) = true + end + + class ScriptedHealth + def initialize(results) + @results = results.dup + end + def reset_tick_cache!; end + def probe(**) + r = @results.shift || @results.last || + Hive::Daemon::AutoRetry::Health::ProbeSetResult.new( + healthy: false, fingerprint: nil, probes: [], rationale: "done", correlation_id: "x" + ) + r + end + end + + def setup + @root = Dir.mktmpdir("hive-auto-retry-int") + @logger = FakeLogger.new + @queue = RecordingQueue.new + @state = Hive::Daemon::AutoRetry::State.new(path: File.join(@root, "daemon_auto_retry.json")) + end + + def teardown + FileUtils.rm_rf(@root) + end + + def make_worktree + wt = File.join(@root, "wt-#{SecureRandom.hex(3)}") + system("git", "init", "-q", wt, exception: true) + system("git", "-C", wt, "config", "user.email", "t@e.com", exception: true) + system("git", "-C", wt, "config", "user.name", "t", exception: true) + File.write(File.join(wt, "f"), "x\n") + system("git", "-C", wt, "add", "f", exception: true) + system("git", "-C", wt, "commit", "-qm", "i", exception: true) + wt + end + + def make_task_row(reason:, message:, stage: "4-execute", agent_cfg: nil) + folder = File.join(@root, "tasks", "task-#{SecureRandom.hex(3)}") + FileUtils.mkdir_p(File.join(folder, "logs")) + File.write(File.join(folder, "logs", "agent.log"), "#{message}\n") + state = File.join(folder, "task.md") + File.write(state, "\n") + wt = make_worktree + Row.new( + project: "demo", + slug: "task-slug-260601-abcd", + stage: stage, + workflow: "coding", + marker: "error", + marker_attrs: { "reason" => reason, "marker_id" => "mid42", "message" => message }, + folder: folder, + state_file: state, + live_task_lock: false, + worktree_path: wt, + id: "58" + ) + end + + def coord(health_results:, enabled: true, cfg: nil) + cfg ||= { "execute" => { "agent" => "codex" }, "claude" => { "mode" => "tmux" } } + Hive::Daemon::AutoRetry::Coordinator.new( + logger: @logger, + controller: FakeController.new, + state: @state, + health: ScriptedHealth.new(health_results), + request_queue: @queue, + enabled: enabled, + config_for_project: ->(_) { cfg }, + project_enabled: ->(_) { true }, + legacy_projects: {} + ) + end + + def healthy(fp) + Hive::Daemon::AutoRetry::Health::ProbeSetResult.new( + healthy: true, fingerprint: fp, probes: [ { "name" => "ok", "ok" => true } ], + rationale: "healthy", correlation_id: "c" + ) + end + + def unhealthy + Hive::Daemon::AutoRetry::Health::ProbeSetResult.new( + healthy: false, fingerprint: nil, probes: [ { "name" => "login", "ok" => false } ], + rationale: "codex_login_unhealthy", correlation_id: "c" + ) + end + + def test_task58_style_parks_while_unhealthy_then_retries + row = make_task_row( + reason: "implementer_failed", + message: "unexpected status 401 Unauthorized: missing bearer authentication" + ) + c = coord(health_results: [ unhealthy, healthy("fp-a") ]) + + c.tick([ row ], now: Time.utc(2026, 7, 17, 12, 0, 0)) + assert_empty @queue.requests, "must not clear while login/smoke fail" + + c.tick([ row ], now: Time.utc(2026, 7, 17, 12, 30, 0)) + assert_equal 1, @queue.requests.size + assert_equal "auto_retry", @queue.requests.first[:requestor] + assert_includes @queue.requests.first[:argv], "markers" + assert_equal 1, @queue.sequences.size + end + + def test_kill_switch_disables_all_actions + row = make_task_row( + reason: "implementer_failed", + message: "HTTP 401 Unauthorized missing bearer authentication" + ) + c = coord(health_results: [ healthy("fp-a") ], enabled: false) + c.tick([ row ]) + assert_empty @queue.requests + end + + def test_second_failure_same_fingerprint_stays_parked + row = make_task_row( + reason: "implementer_failed", + message: "HTTP 401 Unauthorized missing bearer authentication" + ) + c = coord(health_results: [ healthy("fp-a"), healthy("fp-a"), healthy("fp-a") ]) + + c.tick([ row ]) + assert_equal 1, @queue.requests.size + # Simulate attempt accounted + record = @state.load_all.values.first + @state.upsert!( + project: "demo", task_key: "58", stage: "4-execute", + reason: "implementer_failed", phase: "retry_queued" + ) + @state.mark_dispatched!( + project: "demo", task_key: "58", stage: "4-execute", + reason: "implementer_failed", fingerprint: "fp-a", recovery_id: record["recovery_id"] + ) + @queue.requests.clear + @queue.sequences.clear + + attrs = row.marker_attrs.merge("marker_id" => "mid43") + File.write(row.state_file, "\n") + failed_row = Row.new(**row.to_h.merge(marker_attrs: attrs)) + c.tick([ failed_row ]) + assert_equal "failed", @state.load_all.values.first["phase"] + c.tick([ failed_row ]) + assert_empty @queue.requests, "unchanged fingerprint must not re-queue" + end + + def test_claude_launcher_task287_style + row = make_task_row( + reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready in tmux session x", + stage: "3-plan" + ) + File.write(File.join(row.folder, "plan.md"), "\n") + cfg = { + "execute" => { "agent" => "claude" }, + "plan" => { "agent" => "claude" }, + "claude" => { "mode" => "tmux" } + } + c = coord(health_results: [ healthy("fp-claude") ], cfg: cfg) + c.tick([ row ]) + assert_equal 1, @queue.requests.size + end + + def test_dirty_worktree_never_clears + row = make_task_row( + reason: "implementer_failed", + message: "HTTP 401 Unauthorized missing bearer authentication" + ) + File.write(File.join(row.worktree_path, "dirty"), "user edit\n") + c = coord(health_results: [ healthy("fp-a") ]) + c.tick([ row ]) + assert_empty @queue.requests + end + + def test_classifier_rejects_generic_exit_1 + result = Hive::Daemon::AutoRetry::Classifier.classify( + row: { + marker: "error", + marker_attrs: { "reason" => "exit_code", "exit_code" => "1" }, + stage: "4-execute" + }, + config: { "execute" => { "agent" => "codex" } }, + evidence: { text: "exit 1", kind: :log, fresh: true, marker_signature_match: true }, + execute_agent: "codex" + ) + refute result.eligible? + end + + def test_real_queue_recovers_markerless_crash_window_after_restart + row = make_task_row( + reason: "implementer_failed", + message: "HTTP 401 Unauthorized: missing bearer authentication" + ) + old_home = ENV["HIVE_HOME"] + ENV["HIVE_HOME"] = @root + real_state = Hive::Daemon::AutoRetry::State.new(path: File.join(@root, "real-state.json")) + build = lambda do + Hive::Daemon::AutoRetry::Coordinator.new( + logger: @logger, + controller: FakeController.new, + state: real_state, + health: ScriptedHealth.new([ healthy("fp-real") ]), + request_queue: Hive::Daemon::DispatchRequestQueue, + enabled: true, + config_for_project: ->(_) { { "execute" => { "agent" => "codex" } } }, + project_enabled: ->(_) { true }, + legacy_projects: {} + ) + end + + build.call.tick([ row ]) + record = real_state.load_all.values.first + clear_id = record["clear_request_id"] + assert Hive::Daemon::DispatchRequestQueue.request_exists?(clear_id, state_home: @root) + + Hive::Daemon::DispatchRequestQueue.remove(clear_id, state_home: @root) + real_state.upsert!(project: record["project"], task_key: record["task_key"], + stage: record["stage"], reason: record["reason"], phase: "marker_cleared") + File.write(row.state_file, "") + markerless = Row.new(**row.to_h.merge(marker: "none", marker_attrs: {})) + + build.call.tick([ markerless ]) + + pending = Hive::Daemon::DispatchRequestQueue.pending(state_home: @root) + assert_equal 1, pending.size + assert_equal "rerun", pending.first.recovery_step + assert_equal record["recovery_id"], pending.first.recovery_id + ensure + old_home.nil? ? ENV.delete("HIVE_HOME") : ENV["HIVE_HOME"] = old_home + end +end diff --git a/test/integration/gem_package_scripts_test.rb b/test/integration/gem_package_scripts_test.rb index ff2ba7e5..b79a2c4c 100644 --- a/test/integration/gem_package_scripts_test.rb +++ b/test/integration/gem_package_scripts_test.rb @@ -2,6 +2,7 @@ require "test_helper" require "open3" require "rubygems/package" require "tmpdir" +require "rbconfig" class GemPackageScriptsTest < Minitest::Test ROOT = File.expand_path("../..", __dir__) @@ -33,6 +34,30 @@ class GemPackageScriptsTest < Minitest::Test referenced_scripts.each do |path| assert_includes packaged_files, path end + + + extracted = File.join(dir, "installed") + FileUtils.mkdir_p(extracted) + Gem::Package.new(gem_path).extract_files(extracted) + check = <<~'RUBY' + require "json" + require "hive/claude_launcher" + wrapper = Hive::ClaudeLauncher.interactive_wrapper_path + puts JSON.generate( + wrapper_exists: File.file?(wrapper), + wrapper_executable: File.executable?(wrapper), + readiness: Hive::ClaudeLauncher.readiness_detector_self_check + ) + RUBY + out, err, installed_status = Open3.capture3( + RbConfig.ruby, "-I#{File.join(extracted, 'lib')}", "-e", check, + chdir: extracted + ) + assert installed_status.success?, "installed package readiness failed: #{err}" + readiness = JSON.parse(out) + assert_equal true, readiness["wrapper_exists"] + assert_equal true, readiness["wrapper_executable"] + assert_equal true, readiness.dig("readiness", "ok") end end diff --git a/test/integration/markers_command_test.rb b/test/integration/markers_command_test.rb index 19e14bfe..8b89e4a5 100644 --- a/test/integration/markers_command_test.rb +++ b/test/integration/markers_command_test.rb @@ -3,6 +3,7 @@ require "json" require "hive/commands/init" require "hive/commands/new" require "hive/commands/markers" +require "hive/daemon/auto_retry/state" # Integration coverage for `hive markers clear FOLDER --name `. # @@ -411,6 +412,30 @@ class MarkersCommandTest < Minitest::Test [ project, review, File.basename(review) ] end + def seed_plan_auto_retry(dir) + capture_io { Hive::Commands::Init.new(dir).call } + project = File.basename(dir) + capture_io { Hive::Commands::New.new(project, "auto retry marker probe").call } + inbox = Dir[File.join(dir, ".hive-state", "stages", "1-inbox", "*")].first + plan = File.join(dir, ".hive-state", "stages", "3-plan", File.basename(inbox)) + FileUtils.mkdir_p(File.dirname(plan)) + FileUtils.mv(inbox, plan) + task = Hive::Task.new(plan) + File.write(task.state_file, "") + Hive::Markers.set( + task.state_file, :error, + reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready" + ) + recovery_id = "ar-marker-command" + Hive::Daemon::AutoRetry::State.new.upsert!( + project: project, task_key: task.id.to_s, stage: "3-plan", + reason: "claude_launch_failed", phase: "clear_queued", + recovery_id: recovery_id + ) + [ project, plan, task, recovery_id ] + end + def test_match_attr_clears_when_value_matches with_tmp_global_config do with_tmp_git_repo do |dir| @@ -427,6 +452,82 @@ class MarkersCommandTest < Minitest::Test marker = Hive::Markers.current(state) assert_equal :none, marker.name, "matching --match-attr value must allow the clear" + audit = File.readlines(File.join(folder, "events.jsonl"), chomp: true) + .map { |line| JSON.parse(line) } + .find { |event| event["event_type"] == "auto_retry_decision" } + refute_nil audit + assert_equal "manual_rearm", audit.dig("details", "action") + assert_equal "manual_clear", audit.dig("details", "rationale") + end + end + end + + def test_manual_clear_keeps_marker_when_retry_ledger_cannot_rearm + with_tmp_global_config do + with_tmp_git_repo do |dir| + _, folder, _slug = seed_error_with_attrs( + dir, marker_attrs: { reason: "exit_code", exit_code: 143 } + ) + state_file = File.join(folder, "task.md") + failing_state = Object.new + failing_state.define_singleton_method(:rearm_manual!) do |**| + raise Hive::Daemon::AutoRetry::State::Suspended, "disk unavailable" + end + + _out, _err, status = with_replaced_singleton_method( + Hive::Daemon::AutoRetry::State, :new, ->(**) { failing_state } + ) do + with_captured_exit do + Hive::Commands::Markers.new( + "clear", folder, name: "ERROR", match_attr: "exit_code=143" + ).call + end + end + + assert_equal Hive::ExitCodes::SOFTWARE, status + assert_equal :error, Hive::Markers.current(state_file).name + end + end + end + + def test_automatic_clear_advances_ledger_before_removing_marker + with_tmp_global_config do + with_tmp_git_repo do |dir| + project, folder, task, recovery_id = seed_plan_auto_retry(dir) + + capture_io do + Hive::Commands::Markers.new( + "clear", folder, name: "ERROR", recovery_id: recovery_id + ).call + end + + assert_equal :none, Hive::Markers.current(task.state_file).name + record = Hive::Daemon::AutoRetry::State.new.get( + project: project, task_key: task.id.to_s, stage: "3-plan", + reason: "claude_launch_failed" + ) + assert_equal "marker_cleared", record["phase"] + assert_equal recovery_id, record["recovery_id"] + end + end + end + + def test_automatic_clear_rechecks_global_kill_switch + with_tmp_global_config do |home| + with_tmp_git_repo do |dir| + _project, folder, task, recovery_id = seed_plan_auto_retry(dir) + global = YAML.safe_load(File.read(File.join(home, "config.yml"))) + global["daemon"] = { "auto_retry" => { "enabled" => false } } + File.write(File.join(home, "config.yml"), global.to_yaml) + + _out, _err, status = with_captured_exit do + Hive::Commands::Markers.new( + "clear", folder, name: "ERROR", recovery_id: recovery_id + ).call + end + + assert_equal Hive::ExitCodes::WRONG_STAGE, status + assert_equal :error, Hive::Markers.current(task.state_file).name end end end diff --git a/test/unit/cli_test.rb b/test/unit/cli_test.rb index dafa1c29..42374d47 100644 --- a/test/unit/cli_test.rb +++ b/test/unit/cli_test.rb @@ -62,6 +62,20 @@ class HiveCliTest < Minitest::Test out, _err = capture_io { Hive::CLI.start([ "version" ]) } assert_equal "#{Hive::VERSION}\n", out + + json_out, _json_err = capture_io { Hive::CLI.start([ "version", "--json" ]) } + identity = JSON.parse(json_out) + assert_equal Hive::VERSION, identity["version"] + assert_equal Hive::RuntimeIdentity.code_fingerprint, identity["code_fingerprint"] + assert identity.key?("binary_path") + end + + def test_runtime_identity_rejects_missing_cli_or_daemon_path + refute Hive::RuntimeIdentity.matches_cli?( + cli_version: Hive::RuntimeIdentity.version, + cli_fingerprint: Hive::RuntimeIdentity.code_fingerprint, + cli_binary_path: nil + ) end def test_init_forget_prune_update_uninstall_and_migrate_pass_options diff --git a/test/unit/commands/doctor_test.rb b/test/unit/commands/doctor_test.rb index 946169f2..a593170a 100644 --- a/test/unit/commands/doctor_test.rb +++ b/test/unit/commands/doctor_test.rb @@ -263,6 +263,29 @@ class HiveCommandsDoctorTest < Minitest::Test end end + def test_required_agent_rows_accepts_prebounded_dependency_without_cli_probe + with_fake_home do |home| + install_brainstorm_and_plan_skills(home) + dependency = { + kind: "dependency", stage: "claude", label: "claude/tmux", + agent: "tmux", configured_skill: "tmux >= 3.0", skill: "tmux", + status: "present", message: "bounded probe" + } + cfg = base_config("claude" => { "mode" => "tmux" }) + + rows = with_replaced_singleton_method( + Hive::ClaudeLauncher, :tmux_status, -> { raise "must not run" } + ) do + Hive::Commands::Doctor.required_agent_rows( + config: cfg, project_root: nil, dependency_rows: [ dependency ] + ) + end + + assert_equal dependency, rows.first + assert_equal %w[brainstorm plan], rows.drop(1).map { |row| row[:stage] } + end + end + def test_tmux_dependency_row_fails_when_global_tmux_mode_cannot_run_tmux with_fake_home do |home| write_file("#{home}/.claude/plugins/cache/mp/compound-engineering/3.0.1/skills/ce-brainstorm/SKILL.md") diff --git a/test/unit/daemon/auto_retry/classifier_test.rb b/test/unit/daemon/auto_retry/classifier_test.rb new file mode 100644 index 00000000..6fabcdd2 --- /dev/null +++ b/test/unit/daemon/auto_retry/classifier_test.rb @@ -0,0 +1,445 @@ +# frozen_string_literal: true + +require "test_helper" +require "fileutils" +require "hive/daemon/auto_retry/classifier" +require "hive/daemon/auto_retry/evidence" +require "hive/task_action" + +class DaemonAutoRetryClassifierTest < Minitest::Test + FIXTURES = File.expand_path("../../../fixtures/auto_retry", __dir__) + + def classifier(**kwargs) + Hive::Daemon::AutoRetry::Classifier.classify(**kwargs) + end + + def row(stage: "4-execute", reason: "implementer_failed", message: nil, marker_id: "abc123", + marker: "error", attrs: nil) + base = { "reason" => reason, "marker_id" => marker_id } + base["message"] = message if message + base.merge!(attrs) if attrs + { + marker: marker, + marker_attrs: base, + stage: stage, + project: "demo", + slug: "task-slug-260512-abcd", + folder: "/tmp/fake" + } + end + + def codex_config + { "execute" => { "agent" => "codex" }, "claude" => { "mode" => "tmux" } } + end + + def claude_config(mode: "tmux") + { + "execute" => { "agent" => "claude" }, + "brainstorm" => { "agent" => "claude" }, + "plan" => { "agent" => "claude" }, + "claude" => { "mode" => mode } + } + end + + def evidence_from_fixture(*parts) + path = File.join(FIXTURES, *parts) + text = File.read(path) + { text: text, kind: :log, fresh: true, marker_signature_match: true } + end + + # ── Codex auth ────────────────────────────────────────────────────────── + + def test_task58_missing_bearer_classifies_as_codex_auth + result = classifier( + row: row(message: "unexpected status 401 Unauthorized: missing bearer authentication"), + config: codex_config, + evidence: evidence_from_fixture("task-58", "agent.log"), + execute_agent: "codex" + ) + + assert result.eligible? + assert_equal :codex_auth_401, result.recovery_class + end + + def test_task58_basic_auth_variant_classifies + result = classifier( + row: row, + config: codex_config, + evidence: evidence_from_fixture("task-58", "basic-auth.log"), + execute_agent: "codex" + ) + + assert result.eligible? + assert_equal :codex_auth_401, result.recovery_class + end + + def test_generic_exit_code_1_is_ineligible + result = classifier( + row: row(attrs: { "exit_code" => "1", "reason" => "exit_code" }), + config: codex_config, + evidence: evidence_from_fixture("near_miss", "generic-exit-1.log"), + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "unknown_reason", result.rationale + end + + def test_unrelated_401_is_ineligible + result = classifier( + row: row, + config: codex_config, + evidence: evidence_from_fixture("near_miss", "unrelated-401.log"), + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "unknown_signature", result.rationale + end + + def test_non_auth_401_text_is_ineligible + result = classifier( + row: row, + config: codex_config, + evidence: { text: "HTTP 401 from internal metrics service", kind: :log, + fresh: true, marker_signature_match: true }, + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "unknown_signature", result.rationale + end + + def test_unauthorized_auth_wording_without_http_401_is_ineligible + result = classifier( + row: row(message: "Unauthorized: missing bearer authentication"), + config: codex_config, + evidence: { text: "Unauthorized: missing bearer authentication", kind: :marker, + fresh: true, marker_signature_match: true }, + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "unknown_signature", result.rationale + end + + def test_claude_execute_profile_rejects_codex_auth + result = classifier( + row: row(message: "401 Unauthorized missing bearer authentication"), + config: claude_config, + evidence: evidence_from_fixture("task-58", "agent.log"), + execute_agent: "claude" + ) + + refute result.eligible? + assert_equal "wrong_provider", result.rationale + end + + def test_missing_diagnostics_ineligible + result = classifier( + row: row, + config: codex_config, + evidence: { text: "", kind: nil, fresh: true, marker_signature_match: true }, + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "missing_diagnostic", result.rationale + end + + def test_stale_diagnostic_ineligible + result = classifier( + row: row(message: "401 Unauthorized missing bearer authentication"), + config: codex_config, + evidence: { text: "401 missing bearer", kind: :red_status, + fresh: false, marker_signature_match: false }, + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "stale_diagnostic", result.rationale + end + + def test_wrong_stage_for_codex_auth + result = classifier( + row: row(stage: "3-plan", message: "401 Unauthorized missing bearer authentication"), + config: codex_config, + evidence: evidence_from_fixture("task-58", "agent.log"), + execute_agent: "codex" + ) + + refute result.eligible? + assert_equal "wrong_stage", result.rationale + end + + # ── Claude launcher ───────────────────────────────────────────────────── + + def test_task287_launcher_fixture_classifies + result = classifier( + row: row(stage: "3-plan", reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready"), + config: claude_config, + evidence: evidence_from_fixture("task-287", "launcher.log"), + stage_agent: "claude", + claude_mode: "tmux" + ) + + assert result.eligible? + assert_equal :claude_launcher, result.recovery_class + end + + def test_claude_session_exists_excluded + result = classifier( + row: row(stage: "4-execute", reason: "claude_launch_failed", + message: "tmux session hive-foo already exists"), + config: claude_config, + evidence: evidence_from_fixture("near_miss", "claude-session-exists.log"), + stage_agent: "claude", + claude_mode: "tmux" + ) + + refute result.eligible? + assert_equal "excluded_signature", result.rationale + end + + def test_claude_quota_message_excluded + result = classifier( + row: row(stage: "4-execute", reason: "claude_launch_failed"), + config: claude_config, + evidence: evidence_from_fixture("near_miss", "claude-quota.log"), + stage_agent: "claude", + claude_mode: "tmux" + ) + + refute result.eligible? + assert_equal "excluded_signature", result.rationale + end + + def test_unrelated_claude_launch_failed_ineligible + result = classifier( + row: row(stage: "4-execute", reason: "claude_launch_failed", + message: "AgentError: mysterious failure XYZ"), + config: claude_config, + evidence: { text: "AgentError: mysterious failure XYZ", kind: :marker, + fresh: true, marker_signature_match: true }, + stage_agent: "claude", + claude_mode: "tmux" + ) + + refute result.eligible? + assert_equal "unknown_signature", result.rationale + end + + def test_bare_claude_wrapper_name_is_ineligible + result = classifier( + row: row(stage: "3-plan", reason: "claude_launch_failed", + message: "interactive_claude_wrapper"), + config: claude_config, + evidence: { text: "interactive_claude_wrapper", kind: :marker, + fresh: true, marker_signature_match: true }, + stage_agent: "claude", + claude_mode: "tmux" + ) + + refute result.eligible? + assert_equal "unknown_signature", result.rationale + end + + def test_headless_claude_mode_rejects_launcher + result = classifier( + row: row(stage: "4-execute", reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready"), + config: claude_config(mode: "headless"), + evidence: evidence_from_fixture("task-287", "launcher.log"), + stage_agent: "claude", + claude_mode: "headless" + ) + + refute result.eligible? + assert_equal "wrong_claude_mode", result.rationale + end + + def test_unsupported_stage_for_claude_launcher + result = classifier( + row: row(stage: "8-finalize", reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready"), + config: claude_config, + evidence: evidence_from_fixture("task-287", "launcher.log"), + stage_agent: "claude", + claude_mode: "tmux" + ) + + refute result.eligible? + assert_equal "stage_not_supported", result.rationale + end + + def test_brainstorm_claude_launcher_eligible + result = classifier( + row: row(stage: "2-brainstorm", reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready"), + config: claude_config, + evidence: evidence_from_fixture("task-287", "launcher.log"), + stage_agent: "claude", + claude_mode: "tmux" + ) + + assert result.eligible? + assert_equal :claude_launcher, result.recovery_class + end + + # ── Evidence loader fail-closed ───────────────────────────────────────── + + def test_evidence_missing_folder_fails_closed + ev = Hive::Daemon::AutoRetry::Evidence.load(folder: "/nonexistent/path") + assert_equal :missing_folder, ev.error + end + + def test_evidence_loads_log_and_classifies + Dir.mktmpdir("hive-auto-retry-ev") do |folder| + logs = File.join(folder, "logs") + FileUtils.mkdir_p(logs) + File.write(File.join(logs, "run.log"), File.read(File.join(FIXTURES, "task-58", "agent.log"))) + + state = File.join(folder, "task.md") + File.write(state, "\n") + + marker = Hive::Markers.current(state) + ev = Hive::Daemon::AutoRetry::Evidence.load( + folder: folder, state_file: state, marker: marker + ) + refute_nil ev.text + assert_nil ev.error + + result = classifier( + row: row(message: "HTTP 401 Unauthorized missing bearer authentication"), + config: codex_config, + evidence: ev.to_h, + execute_agent: "codex" + ) + assert result.eligible? + end + end + + def test_stale_red_status_signature_fails_closed + Dir.mktmpdir("hive-auto-retry-stale") do |folder| + diag = File.join(folder, "diagnostics") + FileUtils.mkdir_p(diag) + state = File.join(folder, "task.md") + File.write(state, "\n") + marker = Hive::Markers.current(state) + wrong_sig = "0" * 64 + File.write(File.join(diag, "red-status.md"), <<~MD) + --- + summary: 401 Unauthorized missing bearer authentication + generated_by: codex + marker_signature: #{wrong_sig} + --- + body + MD + + # Make red-status newer than state so only signature mismatches + FileUtils.touch(File.join(diag, "red-status.md"), mtime: Time.now + 5) + + ev = Hive::Daemon::AutoRetry::Evidence.load( + folder: folder, state_file: state, marker: marker + ) + assert_equal :stale_diagnostic, ev.error + + result = classifier( + row: row(message: "401 Unauthorized missing bearer authentication"), + config: codex_config, + evidence: ev.to_h, + execute_agent: "codex" + ) + refute result.eligible? + assert_equal "stale_diagnostic", result.rationale + end + end + + def test_unscoped_oversized_log_does_not_raise_or_become_evidence + Dir.mktmpdir("hive-auto-retry-big") do |folder| + logs = File.join(folder, "logs") + FileUtils.mkdir_p(logs) + # Write a large-ish log; loader must bound and not raise. + File.write(File.join(logs, "big.log"), ("x" * 100) + "\n401 Unauthorized missing bearer authentication\n") + + ev = Hive::Daemon::AutoRetry::Evidence.load(folder: folder) + assert_equal :missing_diagnostic, ev.error + end + end + + def test_symlink_evidence_fails_closed + Dir.mktmpdir("hive-auto-retry-sym") do |folder| + outside = File.join(folder, "outside.log") + File.write(outside, "401 Unauthorized missing bearer authentication\n") + logs = File.join(folder, "logs") + FileUtils.mkdir_p(logs) + # Symlink escapes should not be accepted as contained evidence when + # the realpath is still under folder (this one is inside folder but + # tests the File.file? / realpath path). Create a FIFO-like refusal + # via a directory named like a log. + FileUtils.mkdir_p(File.join(logs, "not-a-file.log")) + + ev = Hive::Daemon::AutoRetry::Evidence.load(folder: folder) + # No valid log evidence → missing + assert_equal :missing_diagnostic, ev.error + end + end + + def test_unscoped_invalid_utf8_log_does_not_raise_or_become_evidence + Dir.mktmpdir("hive-auto-retry-utf8") do |folder| + logs = File.join(folder, "logs") + FileUtils.mkdir_p(logs) + File.open(File.join(logs, "bad.log"), "wb") do |f| + f.write("401 Unauthorized missing bearer authentication\n".b) + f.write("\xFF\xFE bad bytes\n".b) + end + + ev = Hive::Daemon::AutoRetry::Evidence.load(folder: folder) + assert_equal :missing_diagnostic, ev.error + end + end + + def test_old_log_from_prior_marker_episode_is_not_used + Dir.mktmpdir("hive-auto-retry-old-log") do |folder| + logs = File.join(folder, "logs") + FileUtils.mkdir_p(logs) + log = File.join(logs, "run.log") + File.write(log, "HTTP 401 Unauthorized missing bearer authentication\n") + FileUtils.touch(log, mtime: Time.now - 601) + state = File.join(folder, "task.md") + File.write(state, "\n") + + ev = Hive::Daemon::AutoRetry::Evidence.load(folder: folder, state_file: state, + marker_attrs: { "marker_id" => "current" }) + assert_equal :marker, ev.kind + refute_includes ev.text, "401" + + result = classifier(row: row(message: "agent failed", marker_id: "current"), + config: codex_config, evidence: ev.to_h, execute_agent: "codex") + refute result.eligible? + end + end + + def test_unknown_reason_audit_rationale + result = classifier( + row: row(reason: "dirty_worktree"), + config: codex_config, + evidence: { text: "dirty", kind: :marker, fresh: true, marker_signature_match: true }, + execute_agent: "codex" + ) + refute result.eligible? + assert_equal "unknown_reason", result.rationale + end + + def test_not_error_marker + result = classifier( + row: row(marker: "review_error", reason: "implementer_failed"), + config: codex_config, + evidence: evidence_from_fixture("task-58", "agent.log"), + execute_agent: "codex" + ) + refute result.eligible? + assert_equal "not_error_marker", result.rationale + end +end diff --git a/test/unit/daemon/auto_retry/coordinator_test.rb b/test/unit/daemon/auto_retry/coordinator_test.rb new file mode 100644 index 00000000..0a71f64d --- /dev/null +++ b/test/unit/daemon/auto_retry/coordinator_test.rb @@ -0,0 +1,362 @@ +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" +require "fileutils" +require "hive/daemon/auto_retry/coordinator" +require "hive/daemon/auto_retry/state" +require "hive/daemon/auto_retry/health" +require "hive/daemon/status_consumer" + +class DaemonAutoRetryCoordinatorTest < Minitest::Test + Row = Hive::Daemon::StatusConsumer::Row + + class FakeLogger + attr_reader :events + def initialize = @events = [] + def event(name, **attrs) = @events << [ name, attrs ] + end + + class FakeController + def running_task?(**) = false + end + + class FakeQueue + attr_reader :requests, :sequences + attr_accessor :fail_sequence + def initialize + @requests = [] + @sequences = {} + @fail_sequence = false + end + def pending = [] + def request_exists?(id, state_home: nil) + @requests.any? { |request| request[:request_id].to_s == id.to_s } + end + def write_request!(**kwargs) + @requests << kwargs + kwargs[:request_id] || "req-#{@requests.size}" + end + def write_sequence!(id, remaining_argvs:, state_home: nil, recovery_id: nil) + raise IOError, "sequence write failed" if @fail_sequence + + @sequences[id] = { argvs: remaining_argvs, recovery_id: recovery_id } + true + end + def remove_if_unclaimed(*) = true + end + + class FakeHealth + attr_reader :probe_count + attr_accessor :healthy, :fingerprint, :signal + + def initialize(healthy: true, fingerprint: "fp-healthy") + @healthy = healthy + @fingerprint = fingerprint + @signal = "signal-#{fingerprint}" + @probe_count = 0 + end + def reset_tick_cache!; end + def signal_fingerprint(**) = @signal + def probe(**) + @probe_count += 1 + Hive::Daemon::AutoRetry::Health::ProbeSetResult.new( + healthy: @healthy, fingerprint: @fingerprint, + probes: [ { "name" => "x", "ok" => @healthy, "timed_out" => false, + "duration_sec" => 0.25, "stdout" => "safe excerpt" } ], + rationale: @healthy ? "healthy" : "probe_failed", correlation_id: "c1" + ) + end + end + + def setup + @logger = FakeLogger.new + @queue = FakeQueue.new + @dir = Dir.mktmpdir("hive-coord") + @state = Hive::Daemon::AutoRetry::State.new(path: File.join(@dir, "state.json")) + end + + def teardown + FileUtils.rm_rf(@dir) + end + + def make_row(reason: "implementer_failed", stage: "4-execute", + message: "HTTP 401 Unauthorized missing bearer authentication", + id: "7", slug: "task-slug-260601-abcd") + folder = File.join(@dir, "task-#{id}") + FileUtils.mkdir_p(File.join(folder, "logs")) + File.write(File.join(folder, "logs", "run.log"), "#{message}\n") + state = File.join(folder, "task.md") + File.write(state, "\n") + wt = File.join(@dir, "wt") + unless File.directory?(wt) + system("git", "init", "-q", wt) + system("git", "-C", wt, "config", "user.email", "t@e.com") + system("git", "-C", wt, "config", "user.name", "t") + File.write(File.join(wt, "f"), "x\n") + system("git", "-C", wt, "add", "f") + system("git", "-C", wt, "commit", "-qm", "i") + end + Row.new( + project: "demo", slug: slug, stage: stage, workflow: "coding", + marker: "error", + marker_attrs: { "reason" => reason, "marker_id" => "mid1", "message" => message }, + folder: folder, state_file: state, live_task_lock: false, worktree_path: wt, id: id + ) + end + + def coordinator(enabled: true, healthy: true, health: nil) + Hive::Daemon::AutoRetry::Coordinator.new( + logger: @logger, + controller: FakeController.new, + state: @state, + health: health || FakeHealth.new(healthy: healthy), + request_queue: @queue, + enabled: enabled, + config_for_project: ->(_) { { "execute" => { "agent" => "codex" }, "claude" => { "mode" => "tmux" } } }, + project_enabled: ->(_) { true }, + legacy_projects: {} + ) + end + + def test_disabled_does_nothing + c = coordinator(enabled: false) + c.tick([ make_row ]) + assert_empty @queue.requests + refute @logger.events.any? { |e| e[0] == :auto_retry_decision && e[1][:action] != "refuse" } + end + + def test_healthy_codex_auth_queues_clear_and_sequence + c = coordinator(healthy: true) + c.tick([ make_row ]) + assert_equal 1, @queue.requests.size + req = @queue.requests.first + assert_equal "auto_retry", req[:requestor] + assert_equal "clear", req[:recovery_step] + assert_includes req[:argv], "markers" + assert_includes req[:argv], "--recovery-id" + assert_equal 1, @queue.sequences.size + seq = @queue.sequences.values.first + assert_equal 1, seq[:argvs].size + refute_includes seq[:argvs].first, "markers" + end + + def test_unhealthy_probe_refuses + c = coordinator(healthy: false) + c.tick([ make_row ]) + assert_empty @queue.requests + decision = @logger.events.find { |e| e[0] == :auto_retry_decision } + assert decision + assert_equal "refuse", decision[1][:action] + end + + def test_unknown_reason_never_queues + c = coordinator + row = make_row(reason: "dirty_worktree", message: "worktree dirty") + c.tick([ row ]) + assert_empty @queue.requests + end + + def test_finalize_never_touched + c = coordinator + row = make_row(stage: "8-finalize", reason: "claude_launch_failed", + message: "claude interactive prompt did not become ready") + c.tick([ row ]) + assert_empty @queue.requests + end + + def test_markerless_restart_recreates_missing_rerun_request + row = make_row + c = coordinator + c.tick([ row ]) + rec = @state.load_all.values.first + @state.upsert!(project: rec["project"], task_key: rec["task_key"], stage: rec["stage"], + reason: rec["reason"], phase: "marker_cleared") + @queue.requests.clear + File.write(row.state_file, "") + markerless = Row.new(**row.to_h.merge(marker: "none", marker_attrs: {})) + + coordinator.tick([ markerless ]) + + assert_equal 1, @queue.requests.size + assert_equal "rerun", @queue.requests.first[:recovery_step] + assert_equal rec["recovery_id"], @queue.requests.first[:recovery_id] + assert_equal "retry_queued", @state.load_all.values.first["phase"] + end + + def test_sequence_failure_never_publishes_runnable_clear + @queue.fail_sequence = true + + coordinator.tick([ make_row ]) + + assert_empty @queue.requests + assert @logger.events.any? { |name, _attrs| name == :auto_retry_sequence_error } + end + + def test_dispatched_attempts_reconcile_to_failed_then_exhausted + row = make_row + key = Hive::Daemon::AutoRetry::State.task_key(task_id: row.id, slug: row.slug) + base = { + project: row.project, task_key: key, stage: row.stage, reason: "implementer_failed", + phase: "retry_dispatched", recovery_id: "ar-first", recovery_class: "codex_auth_401", + slug: row.slug, marker_id: "prior-marker", attempts_dispatched: 1 + } + @state.upsert!(**base) + + coordinator.tick([ row ]) + assert_equal "failed", @state.load_all.values.first["phase"] + + @state.upsert!(**base.merge(phase: "retry_dispatched", recovery_id: "ar-second", + marker_id: "second-prior-marker", attempts_dispatched: 2)) + coordinator.tick([ row ]) + assert_equal "exhausted", @state.load_all.values.first["phase"] + end + + def test_unhealthy_result_waits_for_fallback_cadence + health = FakeHealth.new(healthy: false) + c = coordinator(health: health) + row = make_row + now = Time.utc(2026, 7, 17, 12, 0, 0) + + c.tick([ row ], now: now) + c.tick([ row ], now: now + 60) + + assert_equal 1, health.probe_count + assert_empty @queue.requests + end + + def test_probe_budget_rotates_so_later_candidates_are_not_starved + health = FakeHealth.new(healthy: false) + c = coordinator(health: health) + rows = [ + make_row(id: "1", slug: "task-one-260601-abcd"), + make_row(id: "2", slug: "task-two-260601-abcd"), + make_row(id: "3", slug: "task-three-260601-abcd") + ] + now = Time.utc(2026, 7, 17, 12, 0, 0) + + c.tick(rows, now: now) + assert_equal 2, health.probe_count + c.tick(rows, now: now + 60) + assert_equal 3, health.probe_count + + records = @state.load_all.values + assert_equal 3, records.count { |record| !record["last_probe_at"].to_s.empty? } + end + + def test_task_audit_preserves_false_probe_status_and_bounded_facts + c = coordinator(healthy: false) + row = make_row + + c.tick([ row ]) + + event = File.readlines(File.join(row.folder, "events.jsonl"), chomp: true) + .map { |line| JSON.parse(line) } + .find { |entry| entry["event_type"] == "auto_retry_decision" } + probe = event.dig("details", "probes", 0) + assert_equal false, probe["ok"] + assert_equal false, probe["timed_out"] + assert_equal 0.25, probe["duration_sec"] + assert_equal "safe excerpt", probe["stdout"] + assert_equal row.id, event.dig("details", "task_id") + assert_equal "mid1", event.dig("details", "marker_id") + assert_equal 2, event.dig("details", "max_attempts") + end + + def test_changed_health_fingerprint_reaches_second_attempt_and_exhaustion + health = FakeHealth.new(fingerprint: "fp-a") + c = coordinator(health: health) + row = make_row + t0 = Time.utc(2026, 7, 17, 12, 0, 0) + + c.tick([ row ], now: t0) + first = @state.load_all.values.first + @state.upsert!(project: first["project"], task_key: first["task_key"], + stage: first["stage"], reason: first["reason"], phase: "retry_queued") + @state.mark_dispatched!(project: first["project"], task_key: first["task_key"], + stage: first["stage"], reason: first["reason"], + fingerprint: "fp-a", recovery_id: first["recovery_id"]) + failed_row = row_with_marker_id(row, "mid2") + c.tick([ failed_row ], now: t0 + 1) + assert_equal "failed", @state.load_all.values.first["phase"] + @queue.requests.clear + @queue.sequences.clear + + health.fingerprint = "fp-b" + health.signal = "signal-fp-b" + c.tick([ failed_row ], now: t0 + 2) + assert_empty @queue.requests + assert_equal "fp-b", @state.load_all.values.first["second_candidate_fingerprint"] + + c.tick([ failed_row ], now: t0 + 1_802) + assert_equal 1, @queue.requests.size + second = @state.load_all.values.first + refute_equal first["recovery_id"], second["recovery_id"] + assert_equal 1, second["attempts_dispatched"] + + @state.upsert!(project: second["project"], task_key: second["task_key"], + stage: second["stage"], reason: second["reason"], phase: "retry_queued") + @state.mark_dispatched!(project: second["project"], task_key: second["task_key"], + stage: second["stage"], reason: second["reason"], + fingerprint: "fp-b", recovery_id: second["recovery_id"]) + c.tick([ row_with_marker_id(failed_row, "mid3") ], now: t0 + 1_803) + assert_equal "exhausted", @state.load_all.values.first["phase"] + assert_equal 2, @state.load_all.values.first["attempts_dispatched"] + end + + def test_waiting_marker_is_authoritative_success_after_retry + row = make_row + key = Hive::Daemon::AutoRetry::State.task_key(task_id: row.id, slug: row.slug) + @state.upsert!(project: row.project, task_key: key, stage: row.stage, + reason: "implementer_failed", phase: "retry_dispatched", + recovery_id: "ar-success", recovery_class: "codex_auth_401", + slug: row.slug, marker_id: "mid1", attempts_dispatched: 1) + waiting = Row.new(**row.to_h.merge(marker: "waiting", marker_attrs: {})) + + coordinator.tick([ waiting ]) + + assert_equal "succeeded", @state.load_all.values.first["phase"] + end + + def test_released_retry_request_stays_dispatchable_without_double_accounting + row = make_row + key = Hive::Daemon::AutoRetry::State.task_key(task_id: row.id, slug: row.slug) + retry_id = "retry-pending" + @state.upsert!(project: row.project, task_key: key, stage: row.stage, + reason: "implementer_failed", phase: "retry_dispatched", + recovery_id: "ar-released", recovery_class: "codex_auth_401", + slug: row.slug, marker_id: "mid1", attempts_dispatched: 1, + retry_request_id: retry_id) + @queue.requests << { request_id: retry_id, recovery_step: "rerun" } + markerless = Row.new(**row.to_h.merge(marker: "none", marker_attrs: {})) + + coordinator.tick([ markerless ]) + + record = @state.load_all.values.first + assert_equal "retry_dispatched", record["phase"] + assert_equal 1, record["attempts_dispatched"] + end + + def test_temporarily_absent_status_row_does_not_abort_inflight_recovery + row = make_row + key = Hive::Daemon::AutoRetry::State.task_key(task_id: row.id, slug: row.slug) + @state.upsert!(project: row.project, task_key: key, stage: row.stage, + reason: "implementer_failed", phase: "marker_cleared", + recovery_id: "ar-hidden", slug: row.slug) + + coordinator.tick([]) + + assert_equal "marker_cleared", @state.load_all.values.first["phase"] + end + + private + + def row_with_marker_id(row, marker_id) + attrs = row.marker_attrs.merge("marker_id" => marker_id) + File.write( + row.state_file, + "\n" + ) + Row.new(**row.to_h.merge(marker_attrs: attrs)) + end +end diff --git a/test/unit/daemon/auto_retry/health_test.rb b/test/unit/daemon/auto_retry/health_test.rb new file mode 100644 index 00000000..e36425b2 --- /dev/null +++ b/test/unit/daemon/auto_retry/health_test.rb @@ -0,0 +1,246 @@ +# frozen_string_literal: true + +require "test_helper" +require "hive/daemon/auto_retry/health" +require "hive/daemon/auto_retry/probe_runner" +require "hive/runtime_identity" + +class DaemonAutoRetryHealthTest < Minitest::Test + FakeResult = Struct.new(:ok, :exit_status, :timed_out, :signaled, :stdout, :stderr, + :duration_sec, :error, :argv, keyword_init: true) do + def healthy? + ok == true + end + end + + class FakeRunner + attr_reader :calls + + def initialize(responses) + @responses = responses + @calls = [] + end + + def run(argv, timeout_sec:, chdir: nil, stdin_data: nil) + @calls << { argv: argv, timeout_sec: timeout_sec, chdir: chdir, stdin_data: stdin_data } + key = argv[1..].join(" ") + @responses[key] || @responses[argv.join(" ")] || FakeResult.new(ok: false, error: "unexpected", + exit_status: 1, timed_out: false, + signaled: false, stdout: "", + stderr: "", duration_sec: 0.0, + argv: argv) + end + end + + def with_doctor_green(&block) + rows = [ { label: "plan", status: "present" } ] + with_doctor_rows(rows, &block) + end + + def with_doctor_rows(rows) + original = Hive::Commands::Doctor.method(:required_agent_rows) + Hive::Commands::Doctor.define_singleton_method(:required_agent_rows) { |**| rows } + yield + ensure + Hive::Commands::Doctor.define_singleton_method(:required_agent_rows, original) + end + + def test_codex_login_ok_smoke_fail_is_unhealthy + runner = FakeRunner.new( + "login status" => FakeResult.new(ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: "logged in", stderr: "", duration_sec: 0.1, error: nil, argv: []), + "exec --json --skip-git-repo-check -" => + FakeResult.new(ok: false, exit_status: 1, timed_out: false, signaled: false, + stdout: "", stderr: "auth failed", duration_sec: 0.2, error: "nonzero_exit", argv: []) + ) + health = Hive::Daemon::AutoRetry::Health.new(runner: runner) + result = with_doctor_green do + health.probe_codex({ "execute" => { "agent" => "codex" }, + "agents" => { "codex" => { "bin" => "codex" } } }) + end + refute result.healthy + assert_equal "codex_smoke_unhealthy", result.rationale + end + + def test_codex_healthy_produces_stable_fingerprint + runner = FakeRunner.new( + "login status" => FakeResult.new(ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: "logged in as x", stderr: "", duration_sec: 0.1, error: nil, argv: []), + "exec --json --skip-git-repo-check -" => + FakeResult.new(ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: "{\"type\":\"turn.completed\"}\n", stderr: "", duration_sec: 0.2, + error: nil, argv: []) + ) + health = Hive::Daemon::AutoRetry::Health.new(runner: runner) + cfg = { "execute" => { "agent" => "codex" } } + a, b = with_doctor_green do + first = health.probe_codex(cfg) + health.reset_tick_cache! + [ first, health.probe_codex(cfg) ] + end + assert a.healthy + assert b.healthy + assert_equal a.fingerprint, b.fingerprint + refute_includes a.fingerprint, "token" + smoke_call = runner.calls.find { |call| call[:argv].include?("--skip-git-repo-check") } + assert_equal "Reply with exactly: ok\n", smoke_call[:stdin_data] + assert_equal "-", smoke_call[:argv].last + end + + def test_codex_zero_exit_without_completion_record_is_unhealthy + runner = FakeRunner.new( + "login status" => FakeResult.new(ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: "logged in", stderr: "", duration_sec: 0.1, + error: nil, argv: []), + "exec --json --skip-git-repo-check -" => + FakeResult.new(ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: "{\"type\":\"item.completed\"}\n", stderr: "", + duration_sec: 0.2, error: nil, argv: []) + ) + + result = with_doctor_green do + Hive::Daemon::AutoRetry::Health.new(runner: runner).probe_codex( + { "execute" => { "agent" => "codex" } } + ) + end + + refute result.healthy + assert_equal "codex_smoke_malformed", result.rationale + end + + def test_codex_runs_shared_required_doctor_gate_before_cli_probes + health = Hive::Daemon::AutoRetry::Health.new(runner: FakeRunner.new({})) + rows = [ { label: "plan", status: "missing" } ] + + result = with_doctor_rows(rows) do + health.probe_codex({ "execute" => { "agent" => "codex" } }) + end + + refute result.healthy + assert_equal "doctor_unhealthy", result.rationale + end + + def test_claude_probes_the_configured_runtime_with_a_bound + configured_bin = "/opt/hive-test/claude-custom" + runner = FakeRunner.new( + "-V" => FakeResult.new(ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: "tmux 3.4", stderr: "", duration_sec: 0.1, + error: nil, argv: []), + "--version" => FakeResult.new(ok: false, exit_status: 1, timed_out: false, signaled: false, + stdout: "", stderr: "broken install", duration_sec: 0.1, + error: "nonzero_exit", argv: []) + ) + cfg = { + "plan" => { "agent" => "claude" }, + "agents" => { "claude" => { "bin" => configured_bin } }, + "claude" => { "mode" => "tmux" } + } + + result = with_doctor_green do + Hive::Daemon::AutoRetry::Health.new(runner: runner).probe_claude( + cfg, stage: "3-plan" + ) + end + + refute result.healthy + assert_equal "claude_runtime_unhealthy", result.rationale + call = runner.calls.find { |entry| entry[:argv].first == configured_bin } + refute_nil call + assert_equal Hive::Daemon::AutoRetry::Health::STATUS_TIMEOUT, call[:timeout_sec] + readiness = result.probes.find { |entry| entry["name"] == "readiness_detector" } + assert_equal true, readiness["ok"] + end + + def test_real_probe_runner_drives_fake_codex_with_stdin_json_contract + Dir.mktmpdir("hive-fake-codex") do |dir| + bin = File.join(dir, "codex") + File.write(bin, <<~SH) + #!/bin/sh + if [ "$1 $2" = "login status" ]; then + echo logged-in + exit 0 + fi + input=$(cat) + [ "$input" = "Reply with exactly: ok" ] || exit 9 + printf '%s\n' '{"type":"turn.completed","status":"completed"}' + SH + FileUtils.chmod(0o755, bin) + cfg = { + "execute" => { "agent" => "codex" }, + "agents" => { "codex" => { "bin" => bin } }, + "claude" => { "mode" => "headless" } + } + + result = with_doctor_green do + Hive::Daemon::AutoRetry::Health.new.probe_codex(cfg) + end + + assert result.healthy, result.probes.inspect + assert_equal true, result.probes.last["completion_record"] + end + end + + def test_runtime_identity_snapshot_has_no_secrets + snap = Hive::RuntimeIdentity.snapshot + assert snap.key?("version") + assert snap.key?("code_fingerprint") + refute_includes snap.to_s, "SECRET" + end + + def test_fresh_cli_identity_requires_complete_matching_json + daemon = { + "version" => Hive::VERSION, + "code_fingerprint" => "code-fingerprint", + "binary_path" => "/opt/hive/bin/hive" + } + matching = FakeResult.new( + ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: JSON.generate(daemon), stderr: "", duration_sec: 0.1, + error: nil, argv: [] + ) + incomplete = FakeResult.new( + ok: true, exit_status: 0, timed_out: false, signaled: false, + stdout: JSON.generate("version" => Hive::VERSION), stderr: "", + duration_sec: 0.1, error: nil, argv: [] + ) + + original_snapshot = Hive::RuntimeIdentity.method(:snapshot) + original_binary_path = Hive::RuntimeIdentity.method(:binary_path) + Hive::RuntimeIdentity.define_singleton_method(:snapshot) { |**| daemon } + Hive::RuntimeIdentity.define_singleton_method(:binary_path) { |**| daemon["binary_path"] } + + good = Hive::Daemon::AutoRetry::Health.new( + runner: FakeRunner.new("version --json" => matching) + ).send(:identity_matches?) + bad = Hive::Daemon::AutoRetry::Health.new( + runner: FakeRunner.new("version --json" => incomplete) + ).send(:identity_matches?) + + assert_equal({ ok: true, detail: "match" }, good) + assert_equal({ ok: false, detail: "cli_identity_incomplete" }, bad) + ensure + Hive::RuntimeIdentity.define_singleton_method(:snapshot, original_snapshot) if original_snapshot + Hive::RuntimeIdentity.define_singleton_method(:binary_path, original_binary_path) if original_binary_path + end + + def test_fingerprint_changes_with_config + health = Hive::Daemon::AutoRetry::Health.new(runner: FakeRunner.new({})) + fp1 = health.fingerprint(recovery_class: :codex_auth_401, + config: { "execute" => { "agent" => "codex" } }) + fp2 = health.fingerprint(recovery_class: :codex_auth_401, + config: { "execute" => { "agent" => "claude" } }) + refute_equal fp1, fp2 + end + + def test_fingerprint_canonicalization_preserves_false + health = Hive::Daemon::AutoRetry::Health.new(runner: FakeRunner.new({})) + false_fp = health.fingerprint( + recovery_class: :codex_auth_401, config: {}, extra: { healthy: false } + ) + nil_fp = health.fingerprint( + recovery_class: :codex_auth_401, config: {}, extra: { healthy: nil } + ) + + refute_equal false_fp, nil_fp + end +end diff --git a/test/unit/daemon/auto_retry/policy_test.rb b/test/unit/daemon/auto_retry/policy_test.rb new file mode 100644 index 00000000..77844607 --- /dev/null +++ b/test/unit/daemon/auto_retry/policy_test.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "test_helper" +require "hive/daemon/auto_retry/policy" + +class DaemonAutoRetryPolicyTest < Minitest::Test + P = Hive::Daemon::AutoRetry::Policy + NOW = Time.utc(2026, 6, 1, 12, 0, 0) + + def test_first_retry_immediately_eligible + d = P.decide(record: { "attempts_dispatched" => 0 }, healthy_fingerprint: "fp1", now: NOW) + assert_equal :retry, d.action + assert_equal "first_healthy_signal", d.rationale + assert_equal 1, d.attempt_number + end + + def test_same_fingerprint_blocks_attempt_two + rec = { "attempts_dispatched" => 1, "last_attempted_fingerprint" => "fp1" } + d = P.decide(record: rec, healthy_fingerprint: "fp1", now: NOW) + assert_equal :refuse, d.action + assert_equal "unchanged_fingerprint", d.rationale + end + + def test_different_fingerprint_starts_backoff + rec = { "attempts_dispatched" => 1, "last_attempted_fingerprint" => "fp1" } + d = P.decide(record: rec, healthy_fingerprint: "fp2", now: NOW) + assert_equal :wait, d.action + assert_equal "second_attempt_backoff", d.rationale + assert_equal NOW + P::SECOND_ATTEMPT_BACKOFF_SEC, d.eligible_at + end + + def test_second_attempt_ready_after_window + since = NOW - P::SECOND_ATTEMPT_BACKOFF_SEC + rec = { + "attempts_dispatched" => 1, + "last_attempted_fingerprint" => "fp1", + "second_candidate_fingerprint" => "fp2", + "second_candidate_first_seen_at" => since.iso8601(6) + } + d = P.decide(record: rec, healthy_fingerprint: "fp2", now: NOW) + assert_equal :retry, d.action + assert_equal "second_attempt_ready", d.rationale + end + + def test_exhausted_refuses + rec = { "attempts_dispatched" => 2, "phase" => "exhausted" } + d = P.decide(record: rec, healthy_fingerprint: "fp9", now: NOW) + assert_equal :refuse, d.action + assert_equal "exhausted", d.rationale + end + + def test_negative_throttle + rec = { + "last_negative_signature" => "sig", + "last_negative_at" => NOW.iso8601(6) + } + refute P.should_emit_negative?(record: rec, signature: "sig", now: NOW + 60) + assert P.should_emit_negative?(record: rec, signature: "other", now: NOW + 60) + assert P.should_emit_negative?( + record: rec, signature: "sig", + now: NOW + P::NEGATIVE_DECISION_THROTTLE_SEC + 1 + ) + end +end diff --git a/test/unit/daemon/auto_retry/probe_runner_test.rb b/test/unit/daemon/auto_retry/probe_runner_test.rb new file mode 100644 index 00000000..20d33dba --- /dev/null +++ b/test/unit/daemon/auto_retry/probe_runner_test.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" +require "hive/daemon/auto_retry/probe_runner" + +class DaemonAutoRetryProbeRunnerTest < Minitest::Test + def runner + @runner ||= Hive::Daemon::AutoRetry::ProbeRunner.new(term_grace_sec: 0.2) + end + + def test_successful_command + result = runner.run([ "true" ], timeout_sec: 5) + assert result.healthy? + refute result.timed_out + end + + def test_nonzero_exit_unhealthy + result = runner.run([ "false" ], timeout_sec: 5) + refute result.healthy? + assert_equal "nonzero_exit", result.error + end + + def test_timeout_kills_and_is_unhealthy + result = runner.run([ "sleep", "30" ], timeout_sec: 0.3) + refute result.healthy? + assert result.timed_out + assert_equal "timeout", result.error + end + + def test_missing_executable + result = runner.run([ "/nonexistent/binary-xyz" ], timeout_sec: 2) + refute result.healthy? + assert_equal "missing_executable", result.error + end + + def test_output_capped_and_redacted + r = Hive::Daemon::AutoRetry::ProbeRunner.new(excerpt_bytes: 80) + result = r.run( + [ "bash", "-c", "echo 'Authorization: Bearer sk-secret-token-1234567890'; dd if=/dev/zero bs=1 count=5000 2>/dev/null | tr '\\0' 'X'" ], + timeout_sec: 5 + ) + assert result.healthy?, result.inspect + refute_includes result.stdout, "sk-secret-token-1234567890" + assert result.stdout.bytesize <= 200, "stdout bytes=#{result.stdout.bytesize}" + end + + def test_chdir_isolation + Dir.mktmpdir do |dir| + result = runner.run([ "bash", "-c", "pwd" ], timeout_sec: 5, chdir: dir) + assert result.healthy? + assert_includes result.stdout, dir + end + end + + def test_pipe_filling_successful_child_is_drained_while_running + result = runner.run( + [ "ruby", "-e", '$stdout.write("x" * 262_144); $stderr.write("y" * 262_144)' ], + timeout_sec: 5 + ) + + assert result.healthy?, result.inspect + refute result.timed_out + assert_operator result.stdout.bytesize, :<=, 2_048 + assert_operator result.stderr.bytesize, :<=, 2_048 + end +end diff --git a/test/unit/daemon/auto_retry/safety_test.rb b/test/unit/daemon/auto_retry/safety_test.rb new file mode 100644 index 00000000..da2bb501 --- /dev/null +++ b/test/unit/daemon/auto_retry/safety_test.rb @@ -0,0 +1,203 @@ +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" +require "fileutils" +require "open3" +require "hive/daemon/auto_retry/safety" +require "hive/daemon/status_consumer" +require "hive/markers" + +class DaemonAutoRetrySafetyTest < Minitest::Test + Row = Hive::Daemon::StatusConsumer::Row + Safety = Hive::Daemon::AutoRetry::Safety + + FakeResult = Struct.new(:ok, :timed_out, :stdout, keyword_init: true) do + def healthy? = ok == true + end + + class TimeoutRunner + def run(*) = FakeResult.new(ok: false, timed_out: true, stdout: "") + end + + def make_git_repo + dir = Dir.mktmpdir("hive-safety-wt") + system("git", "init", "-q", dir, exception: true) + system("git", "-C", dir, "config", "user.email", "t@example.com", exception: true) + system("git", "-C", dir, "config", "user.name", "t", exception: true) + File.write(File.join(dir, "README"), "x\n") + system("git", "-C", dir, "add", "README", exception: true) + system("git", "-C", dir, "commit", "-qm", "init", exception: true) + dir + end + + def row_for(folder:, stage: "4-execute", reason: "implementer_failed", marker_id: "m1", + worktree_path: nil, live_task_lock: false, marker: "error") + state = File.join(folder, "task.md") + File.write(state, "\n") unless File.exist?(state) + Row.new( + project: "demo", + slug: "task-slug-260601-abcd", + stage: stage, + workflow: "coding", + marker: marker, + marker_attrs: { "reason" => reason, "marker_id" => marker_id }, + folder: folder, + state_file: state, + live_task_lock: live_task_lock, + worktree_path: worktree_path, + id: "42" + ) + end + + def test_clean_execute_worktree_is_safe + wt = make_git_repo + Dir.mktmpdir("hive-safety-task") do |folder| + row = row_for(folder: folder, worktree_path: wt) + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "implementer_failed") + assert result.safe?, result.rationale + end + ensure + FileUtils.rm_rf(wt) if wt + end + + def test_dirty_execute_worktree_unsafe + wt = make_git_repo + File.write(File.join(wt, "dirty.txt"), "nope\n") + Dir.mktmpdir("hive-safety-task") do |folder| + row = row_for(folder: folder, worktree_path: wt) + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "implementer_failed") + refute result.safe? + assert_equal "dirty_worktree", result.rationale + end + ensure + FileUtils.rm_rf(wt) if wt + end + + def test_blank_plan_is_safe + Dir.mktmpdir("hive-safety-plan") do |folder| + File.write(File.join(folder, "plan.md"), "\n") + row = row_for(folder: folder, stage: "3-plan", reason: "claude_launch_failed") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "claude_launch_failed") + assert result.safe?, result.rationale + end + end + + def test_plan_with_prose_is_unsafe + Dir.mktmpdir("hive-safety-plan") do |folder| + File.write(File.join(folder, "plan.md"), "# Plan\n\nDo the thing.\n") + row = row_for(folder: folder, stage: "3-plan", reason: "claude_launch_failed") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "claude_launch_failed") + refute result.safe? + assert_equal "plan_has_content", result.rationale + end + end + + def test_plan_user_feedback_html_comment_is_unsafe + Dir.mktmpdir("hive-safety-plan") do |folder| + File.write(File.join(folder, "plan.md"), "\n") + row = row_for(folder: folder, stage: "3-plan", reason: "claude_launch_failed") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "claude_launch_failed") + refute result.safe? + assert_equal "plan_has_content", result.rationale + end + end + + def test_brainstorm_user_feedback_html_comment_is_unsafe + Dir.mktmpdir("hive-safety-bs") do |folder| + File.write(File.join(folder, "brainstorm.md"), "\n") + row = row_for(folder: folder, stage: "2-brainstorm", reason: "claude_launch_failed") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "claude_launch_failed") + refute result.safe? + assert_equal "malformed_brainstorm", result.rationale + end + end + + def test_answered_brainstorm_unsafe + Dir.mktmpdir("hive-safety-bs") do |folder| + File.write(File.join(folder, "brainstorm.md"), <<~MD) + ## Round 1 + ### Q1. What? + ### A1. + yes please + MD + row = row_for(folder: folder, stage: "2-brainstorm", reason: "claude_launch_failed") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "claude_launch_failed") + refute result.safe? + assert_equal "answered_questions", result.rationale + end + end + + def test_finalize_is_merge_watcher_owned + Dir.mktmpdir("hive-safety-fin") do |folder| + row = row_for(folder: folder, stage: "8-finalize", reason: "claude_launch_failed") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "claude_launch_failed") + refute result.safe? + assert_equal "merge_watcher_owned", result.rationale + end + end + + def test_live_lock_blocks + wt = make_git_repo + Dir.mktmpdir("hive-safety-lock") do |folder| + row = row_for(folder: folder, worktree_path: wt, live_task_lock: true) + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "implementer_failed") + refute result.safe? + assert_equal "live_task_lock", result.rationale + end + ensure + FileUtils.rm_rf(wt) if wt + end + + def test_marker_id_rotation_blocks + wt = make_git_repo + Dir.mktmpdir("hive-safety-rot") do |folder| + row = row_for(folder: folder, worktree_path: wt, marker_id: "newid") + result = Safety.new.check(row: row, observed_marker_id: "oldid", + observed_reason: "implementer_failed") + refute result.safe? + assert_equal "marker_id_changed", result.rationale + end + ensure + FileUtils.rm_rf(wt) if wt + end + + def test_reread_detects_lock_created_after_status_probe + wt = make_git_repo + Dir.mktmpdir("hive-safety-race") do |folder| + row = row_for(folder: folder, worktree_path: wt) + Hive::Lock.acquire_task_lock(folder, "command" => "run") + result = Safety.new.check(row: row, observed_marker_id: "m1", + observed_reason: "implementer_failed", re_read: true) + refute result.safe? + assert_equal "live_task_lock", result.rationale + ensure + Hive::Lock.release_task_lock(folder) + end + ensure + FileUtils.rm_rf(wt) if wt + end + + def test_git_status_timeout_fails_closed + wt = make_git_repo + Dir.mktmpdir("hive-safety-timeout") do |folder| + row = row_for(folder: folder, worktree_path: wt) + result = Safety.new(runner: TimeoutRunner.new).check( + row: row, observed_marker_id: "m1", observed_reason: "implementer_failed" + ) + refute result.safe? + assert_equal "git_status_timeout", result.rationale + end + ensure + FileUtils.rm_rf(wt) if wt + end +end diff --git a/test/unit/daemon/auto_retry/state_test.rb b/test/unit/daemon/auto_retry/state_test.rb new file mode 100644 index 00000000..bfdf87aa --- /dev/null +++ b/test/unit/daemon/auto_retry/state_test.rb @@ -0,0 +1,143 @@ +# frozen_string_literal: true + +require "test_helper" +require "tmpdir" +require "hive/daemon/auto_retry/state" +require "hive/daemon/auto_retry/policy" + +class DaemonAutoRetryStateTest < Minitest::Test + def queue_retry(state, recovery_id:, phase: "retry_queued", **attrs) + state.upsert!( + project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed", + phase: phase, recovery_id: recovery_id, **attrs + ) + end + + def with_state + Dir.mktmpdir("hive-auto-retry-state") do |dir| + path = File.join(dir, "daemon_auto_retry.json") + state = Hive::Daemon::AutoRetry::State.new(path: path, clock: -> { Time.utc(2026, 6, 1, 12, 0, 0) }) + yield state, path + end + end + + def test_upsert_and_get + with_state do |state, _path| + state.upsert!(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed", + phase: "candidate", marker_id: "abc") + rec = state.get(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed") + assert_equal "candidate", rec["phase"] + assert_equal "abc", rec["marker_id"] + assert_equal 0, rec["attempts_dispatched"] + end + end + + def test_mark_dispatched_increments_attempts + with_state do |state, _path| + queue_retry(state, recovery_id: "ar-1") + state.mark_dispatched!(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed", fingerprint: "fp1", + recovery_id: "ar-1") + rec = state.get(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed") + assert_equal 1, rec["attempts_dispatched"] + assert_equal "fp1", rec["last_attempted_fingerprint"] + assert_equal "retry_dispatched", rec["phase"] + end + end + + def test_marker_id_does_not_create_new_budget + with_state do |state, _path| + queue_retry(state, recovery_id: "ar-1", marker_id: "old") + state.mark_dispatched!(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed", fingerprint: "fp1", + recovery_id: "ar-1") + state.upsert!(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed", + marker_id: "new") + rec = state.get(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed") + assert_equal 1, rec["attempts_dispatched"] + assert_equal "new", rec["marker_id"] + end + end + + def test_manual_rearm_resets_attempts + with_state do |state, _path| + state.upsert!(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed") + 2.times do |i| + queue_retry(state, recovery_id: "ar-#{i}") + state.mark_dispatched!(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed", fingerprint: "fp#{i}", + recovery_id: "ar-#{i}") + end + state.mark_exhausted!(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed") + state.rearm_manual!(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed") + rec = state.get(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed") + assert_equal 0, rec["attempts_dispatched"] + assert_equal "candidate", rec["phase"] + end + end + + def test_corrupt_state_suspends + with_state do |state, path| + File.write(path, "{not json") + loaded = state.load_all + assert_equal({}, loaded) + assert state.suspended? + assert_raises(Hive::Daemon::AutoRetry::State::Suspended) do + state.upsert!(project: "p", task_key: "t", stage: "4-execute", reason: "x") + end + end + end + + def test_structurally_corrupt_records_suspends_without_resetting_budget + with_state do |state, path| + File.write(path, JSON.generate("schema_version" => 1, "records" => [])) + + assert_equal({}, state.load_all) + assert state.suspended? + assert_equal "records_not_hash", state.suspend_reason + assert_raises(Hive::Daemon::AutoRetry::State::Suspended) do + state.upsert!(project: "p", task_key: "t", stage: "4-execute", reason: "x") + end + end + end + + def test_dispatch_accounting_is_idempotent_for_same_recovery + with_state do |state, _path| + queue_retry(state, recovery_id: "ar-stable") + 2.times do + state.mark_dispatched!(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed", fingerprint: "fp1", + recovery_id: "ar-stable") + end + + rec = state.get(project: "p", task_key: "t1", stage: "4-execute", + reason: "implementer_failed") + assert_equal 1, rec["attempts_dispatched"] + assert_equal "retry_dispatched", rec["phase"] + end + end + + def test_newer_schema_suspends_writes + with_state do |state, path| + File.write(path, JSON.generate("schema_version" => 99, "records" => {})) + state.load_all + assert state.suspended? + assert_equal "newer_schema", state.suspend_reason + end + end + + def test_keys_isolated_by_reason + with_state do |state, _path| + state.upsert!(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed", + attempts_dispatched: 1) + state.upsert!(project: "p", task_key: "t1", stage: "4-execute", reason: "claude_launch_failed", + attempts_dispatched: 0) + a = state.get(project: "p", task_key: "t1", stage: "4-execute", reason: "implementer_failed") + b = state.get(project: "p", task_key: "t1", stage: "4-execute", reason: "claude_launch_failed") + assert_equal 1, a["attempts_dispatched"] + assert_equal 0, b["attempts_dispatched"] + end + end +end diff --git a/test/unit/daemon/dispatch_request_queue_test.rb b/test/unit/daemon/dispatch_request_queue_test.rb index a3c6f5e7..5f34ed0c 100644 --- a/test/unit/daemon/dispatch_request_queue_test.rb +++ b/test/unit/daemon/dispatch_request_queue_test.rb @@ -175,6 +175,32 @@ class HiveDaemonDispatchRequestQueueTest < Minitest::Test end end + def test_auto_retry_idempotence_rejects_request_id_collision + Dir.mktmpdir("hive-dispatch-queue") do |dir| + argv = [ "hive", "run", "slug-x", "--json" ] + Q.write_request!( + project: "hive", slug: "slug-x", argv: argv, + requestor: "auto_retry", request_id: "recovery-request", + recovery_id: "ar-one", recovery_step: "rerun", state_home: dir + ) + + assert_equal "recovery-request", Q.write_request!( + project: "hive", slug: "slug-x", argv: argv, + requestor: "auto_retry", request_id: "recovery-request", + recovery_id: "ar-one", recovery_step: "rerun", state_home: dir + ) + error = assert_raises(ArgumentError) do + Q.write_request!( + project: "hive", slug: "slug-x", + argv: [ "hive", "archive", "slug-x", "--json" ], + requestor: "auto_retry", request_id: "recovery-request", + recovery_id: "ar-one", recovery_step: "rerun", state_home: dir + ) + end + assert_includes error.message, "collides with different recovery metadata" + end + end + def test_remove_ignores_empty_request_id Dir.mktmpdir("hive-dispatch-queue") do |dir| refute Q.remove("", state_home: dir) diff --git a/test/unit/daemon/dispatcher_test.rb b/test/unit/daemon/dispatcher_test.rb index d5de2782..bbfd61a2 100644 --- a/test/unit/daemon/dispatcher_test.rb +++ b/test/unit/daemon/dispatcher_test.rb @@ -216,12 +216,14 @@ class HiveDaemonDispatcherTest < Minitest::Test with_patrol_scheduler: false, project_enabled: true, dispatch_state: nil, status_result: nil, dispatch_request_state_home: nil, dispatch_result_state_home: nil, - with_digest_scheduler: false, with_answer_digest_scheduler: false) + with_digest_scheduler: false, with_answer_digest_scheduler: false, + auto_retry_enabled: true) config = { "daemon" => { "edit_debounce_sec" => 30, "poll_interval_sec" => 30, - "shutdown_grace_sec" => 60 + "shutdown_grace_sec" => 60, + "auto_retry" => { "enabled" => auto_retry_enabled } } } controller = Hive::Daemon::ConcurrencyController.new( @@ -2538,7 +2540,8 @@ end Q = Hive::Daemon::DispatchRequestQueue def write_request_file(dir, slug:, request_id:, created_at: T0, argv: nil, project: "p1", - trigger: "answer_complete") + trigger: "answer_complete", requestor: "bot", recovery_id: nil, + recovery_step: nil) argv ||= [ "hive", "run", slug, "--json" ] path = File.join(Q.directory(state_home: dir), Q.filename_for(created_at: created_at, request_id: request_id)) payload = { @@ -2549,11 +2552,13 @@ end "project" => project, "slug" => slug, "argv" => argv, - "requestor" => "bot", + "requestor" => requestor, "chat_id" => 42, "update_id" => 99, "trigger" => trigger } + payload["recovery_id"] = recovery_id if recovery_id + payload["recovery_step"] = recovery_step if recovery_step File.write(path, JSON.generate(payload)) path end @@ -3124,6 +3129,113 @@ end end end + def test_kill_switch_rejects_pending_auto_retry_request + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + dispatcher, sup, _ctrl, logger, _mw = make_dispatcher( + rows: [], dispatch_request_state_home: state_home, auto_retry_enabled: false + ) + write_request_file( + state_home, slug: "s1", request_id: "AUTOOFF", + requestor: "auto_retry", recovery_id: "ar-off", recovery_step: "rerun" + ) + + dispatcher.tick(now: T0) + + rejected = logger.events.find { |name, attrs| name == :dispatch_request_rejected && attrs[:request_id] == "AUTOOFF" } + refute_nil rejected + assert_equal "auto_retry_disabled", rejected[1][:reason] + assert_empty sup.spawned + assert_empty Q.pending(state_home: state_home) + end + end + + def test_auto_retry_request_without_durable_authorization_is_rejected + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + dispatcher, sup, _ctrl, logger, _mw = make_dispatcher( + rows: [], dispatch_request_state_home: state_home + ) + write_request_file( + state_home, slug: "s1", request_id: "AUTOBAD", + requestor: "auto_retry", recovery_id: "ar-missing", recovery_step: "rerun" + ) + + dispatcher.tick(now: T0) + + rejected = logger.events.find { |name, attrs| name == :dispatch_request_rejected && attrs[:request_id] == "AUTOBAD" } + refute_nil rejected + assert_equal "auto_retry_authorization_failed", rejected[1][:reason] + assert_empty sup.spawned + assert_empty Q.pending(state_home: state_home) + end + end + + def test_authorized_auto_retry_rerun_dispatches_and_accounts_once + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + dispatcher, sup, _ctrl, _logger, _mw = make_dispatcher( + rows: [], dispatch_request_state_home: state_home + ) + state = Hive::Daemon::AutoRetry::State.new(path: File.join(state_home, "auto-retry.json")) + dispatcher.instance_variable_get(:@auto_retry).instance_variable_set(:@state, state) + state.upsert!( + project: "p1", task_key: "s1", stage: "4-execute", + reason: "implementer_failed", phase: "retry_queued", + recovery_id: "ar-good", slug: "s1", pending_fingerprint: "fp-good", + retry_request_id: "AUTOGOOD", + rerun_argv: [ "hive", "run", "s1", "--json" ] + ) + write_request_file( + state_home, slug: "s1", request_id: "AUTOGOOD", + requestor: "auto_retry", recovery_id: "ar-good", recovery_step: "rerun" + ) + stub_find_project!(dispatcher, "p1") + begin + dispatcher.tick(now: T0) + + assert_equal 1, sup.spawned.size + assert_equal "AUTOGOOD", sup.spawned.first[:request_id] + record = state.find_by_recovery_id("ar-good") + assert_equal 1, record["attempts_dispatched"] + assert_equal "retry_dispatched", record["phase"] + + dispatcher.tick(now: T0 + 1) + assert_equal 1, sup.spawned.size + assert_equal 1, state.find_by_recovery_id("ar-good")["attempts_dispatched"] + ensure + restore_find_project! + end + end + end + + def test_auto_retry_recovery_id_cannot_authorize_different_argv + Dir.mktmpdir("hive-dispatch-queue") do |state_home| + dispatcher, sup, _ctrl, logger, _mw = make_dispatcher( + rows: [], dispatch_request_state_home: state_home + ) + state = Hive::Daemon::AutoRetry::State.new(path: File.join(state_home, "auto-retry.json")) + dispatcher.instance_variable_get(:@auto_retry).instance_variable_set(:@state, state) + state.upsert!( + project: "p1", task_key: "s1", stage: "4-execute", + reason: "implementer_failed", phase: "retry_queued", + recovery_id: "ar-forged", slug: "s1", retry_request_id: "AUTOFORGED", + rerun_argv: [ "hive", "run", "s1", "--json" ] + ) + write_request_file( + state_home, slug: "s1", request_id: "AUTOFORGED", + argv: [ "hive", "archive", "s1", "--json" ], + requestor: "auto_retry", recovery_id: "ar-forged", recovery_step: "rerun" + ) + + dispatcher.tick(now: T0) + + rejected = logger.events.find do |name, attrs| + name == :dispatch_request_rejected && attrs[:request_id] == "AUTOFORGED" + end + refute_nil rejected + assert_equal "auto_retry_authorization_failed", rejected[1][:reason] + assert_empty sup.spawned + end + end + def test_dispatch_request_blocked_when_in_flight_for_same_slug Dir.mktmpdir("hive-dispatch-queue") do |state_home| dispatcher, sup, ctrl, logger, _mw = make_dispatcher( diff --git a/test/unit/events_test.rb b/test/unit/events_test.rb index 28ac2224..47c99c2f 100644 --- a/test/unit/events_test.rb +++ b/test/unit/events_test.rb @@ -208,4 +208,13 @@ class EventsTest < Minitest::Test File.define_singleton_method(:open, original_open) if original_open end end + + def test_reencoded_truncated_details_stay_within_the_advertised_cap + details = { "quoted" => ('\\"' * (Hive::Events::MAX_DETAILS_BYTES * 2)) } + + bounded = Hive::Events.bound_details(details) + + assert_equal true, bounded["truncated"] + assert_operator JSON.generate(bounded).bytesize, :<=, Hive::Events::MAX_DETAILS_BYTES + end end diff --git a/wiki/commands/doctor.md b/wiki/commands/doctor.md index 81cc8f8b..97b43df5 100644 --- a/wiki/commands/doctor.md +++ b/wiki/commands/doctor.md @@ -3,11 +3,11 @@ title: hive doctor type: command source: lib/hive/commands/doctor.rb, lib/hive/skill_check.rb created: 2026-05-07 -updated: 2026-06-14 +updated: 2026-07-18 tags: [command, preflight, skills, tmux] --- -**TLDR**: `hive doctor` walks `brainstorm` + `plan` stage configs **and** every entry in `review.reviewers[]`, asking each agent profile to verify its configured skill (e.g. `/plan`, `/llm-wiki:wiki-plan`, `/ce-brainstorm`, `/ce-code-review`, `/skill:wiki-plan`) actually resolves to an installed slash-command or skill on disk. When `claude.mode: tmux`, it also checks `tmux >= 3.0`; initialized projects also get a non-fatal `wiki/qmd` dependency row so missing/broken QMD and native Node ABI mismatches are visible. Legacy configs with `brainstorm.runtime` get an advisory warning. Prints a status table; `--json` emits a `hive-doctor.v1` envelope. Also runs **non-fatally** at the end of `hive init` as a preflight: missing skills surface as stderr warnings, but `init` exit code is unaffected. +**TLDR**: `hive doctor` walks `brainstorm` + `plan` stage configs **and** every entry in `review.reviewers[]`, asking each agent profile to verify its configured skill actually resolves on disk. When `claude.mode: tmux`, it also checks `tmux >= 3.0`; QMD and legacy-config rows remain advisory. Prints a table; `--json` emits `hive-doctor.v1`. The daemon auto-retry health gate reuses the lightweight `Doctor.required_agent_rows` API, so required-agent decisions cannot drift from the CLI while optional warnings stay outside recovery eligibility. Doctor also runs **non-fatally** after `hive init`. ## Usage @@ -84,6 +84,15 @@ After `Hive::Commands::Init#call` finishes its summary, it invokes `run_init_pre Rescue scope is `StandardError` (with a `Errno::EPIPE` micro-rescue around `warn`); `Interrupt` and `SystemExit` propagate. Unexpected verifier raises produce a "this may be a hive bug, please report" hint so silent swallow is mitigated. `Doctor#rows` (an `attr_reader`) lets the preflight read probe results in-process without re-running the renderer. +## Daemon required-agent API + +`Doctor.required_agent_rows(config:, project_root:, dependency_rows:)` returns +only required dependency, stage-skill, and reviewer rows. Daemon health passes +already-bounded tmux evidence through `dependency_rows`, avoiding Doctor's +human-oriented process probe and rendering. A `missing` or `version_too_old` +row makes either Codex or Claude auto-retry health red; warning-only QMD and +migration rows do not. + ## Tests - `test/unit/commands/doctor_test.rb` — stage rows, reviewer happy path, mixed agents, empty/nil/absent reviewers, non-agent kinds, pi reviewer rows, QMD managed-binary and broken-binary rows, JSON envelope shape, long-label width, `attr_reader :rows` exposure. diff --git a/wiki/commands/markers.md b/wiki/commands/markers.md index ec76e1e4..c583e6e7 100644 --- a/wiki/commands/markers.md +++ b/wiki/commands/markers.md @@ -3,11 +3,11 @@ title: hive markers type: command source: lib/hive/commands/markers.rb created: 2026-04-26 -updated: 2026-05-27 +updated: 2026-07-18 tags: [command, markers, recovery, json] --- -**TLDR**: `hive markers clear FOLDER --name [--project NAME] [--json]` removes a single recovery marker from a task's state file (atomic write) and records a `hive_commit` so the audit trail stays accurate. Replaces the previous "manually edit `task.md` and delete the marker comment" recovery prose with a deterministic, agent-callable surface. +**TLDR**: `hive markers clear FOLDER --name [--project NAME] [--json]` removes a single recovery marker from a task's state file (atomic write) and records a `hive_commit` so the audit trail stays accurate. Replaces the previous "manually edit `task.md` and delete the marker comment" recovery prose with a deterministic, agent-callable surface. Successful **manual** clears rearm the daemon auto-retry attempt budget for that project/task/stage/reason; automatic clears pass internal `--recovery-id` so they advance the durable phase without rearming (see [[modules/daemon]]). ## Usage @@ -41,9 +41,11 @@ Only recovery markers are clearable. Terminal-success markers (`REVIEW_COMPLETE` 3. Validate the requested `--name` against `Hive::Commands::Markers::ALLOWED_NAMES`. Anything else raises `Hive::WrongStage` (exit 4). 4. Read the current marker via `Hive::Markers.current(state_file)`. If the marker name does NOT match `--name`, raise `Hive::WrongStage` — refusing to silently clear a different state. 5. If `--match-attr` is present, require every supplied `KEY=VALUE` pair to match the current marker. Comma-separated pairs such as `reason=exit_code,exit_code=143` are all checked; any mismatch raises `Hive::WrongStage`. TUI ERROR recovery prefers generated `marker_id` attrs when available and uses observed reason/exit_code attrs for legacy rows. -6. Remove the marker line: `File.read` the body, `sub` out the exact `marker.raw` comment plus its trailing newline (if it sat alone on a line), then `Hive::Markers.write_atomic` the result. Surrounding prose, headings, and other markers stay untouched. -7. Record a `hive_commit` on the `hive/state` branch (`hive: / markers clear `). -8. Emit a stdout summary (or one-line `hive-markers-clear` JSON document with `--json`); print a `next: hive run ` hint to stderr. +6. For an internal automatic clear, acquire the real per-task `.lock`, recheck the global kill switch, require the matching durable recovery ID/phase, and rerun stage safety against the on-disk marker and lock. A runner cannot start inside the final probe-to-clear window. +7. Persist safety-critical accounting **before** marker removal: automatic clear advances to `marker_cleared`; manual clear rearms the two-attempt budget. Any ledger error leaves the marker intact and fails closed. +8. Remove the marker line under `.markers-lock`. Surrounding prose, headings, and other markers stay untouched. +9. Record a `hive_commit` on the `hive/state` branch. Manual clears also emit a task-local `auto_retry_decision` event with `action=manual_rearm`. +10. Emit a stdout summary (or one-line `hive-markers-clear` JSON document with `--json`); print a `next: hive run ` hint to stderr. ## JSON contract (`schema = "hive-markers-clear"`, version 1) diff --git a/wiki/log.d/20260717T221331Z-daemon-health-gated-auto-retry.md b/wiki/log.d/20260717T221331Z-daemon-health-gated-auto-retry.md new file mode 100644 index 00000000..e307f0c2 --- /dev/null +++ b/wiki/log.d/20260717T221331Z-daemon-health-gated-auto-retry.md @@ -0,0 +1,22 @@ +--- +title: Daemon health-gated auto-retry (v1) +date: 2026-07-17T22:13:31Z +tags: [daemon, auto-retry, markers, config, events] +--- + +**Action:** Implemented conservative daemon auto-retry for two exact recovery +classes (`codex_auth_401`, `claude_launcher`) under +`lib/hive/daemon/auto_retry/`. Fail-closed classification, bounded probes with +process-group timeouts, durable two-attempt ledger +(`daemon_auto_retry.json`), stage safety guards, guarded +`markers clear --recovery-id` + same-stage rerun via +`DispatchRequestQueue` (`requestor=auto_retry`), global kill switch +`daemon.auto_retry.enabled` (default true), and dual-surface audit +(`auto_retry_decision` on daemon log + task `events.jsonl`). Merge-watcher +finalize recovery retains precedence. Unit + hermetic integration coverage +under `test/unit/daemon/auto_retry/` and +`test/integration/daemon_auto_retry_test.rb`. + +**Pages:** [[modules/daemon]], [[modules/config]], [[modules/events]], +[[modules/markers]], [[commands/markers]], [[state-model]], [[operating]], +[[testing]]. diff --git a/wiki/log.d/20260718T003000Z-auto-retry-review-hardening.md b/wiki/log.d/20260718T003000Z-auto-retry-review-hardening.md new file mode 100644 index 00000000..aa50d8d1 --- /dev/null +++ b/wiki/log.d/20260718T003000Z-auto-retry-review-hardening.md @@ -0,0 +1,21 @@ +--- +title: Auto-retry review hardening +date: 2026-07-18T00:30:00Z +tags: [daemon, auto-retry, recovery, health, audit] +--- + +**Action:** Hardened health-gated daemon recovery around crash and restart +boundaries. Exact/current-episode classification, production-shaped bounded +Codex/Claude health checks, shared Doctor gating, strict CLI identity, +fail-closed ledger validation, deterministic continuation-first queue +publication, durable request authorization, idempotent dispatch accounting, +authoritative terminal reconciliation, fair 30-minute probe cadence, and a +kill switch that also rejects pending automatic work now protect the two-attempt +lifecycle. Automatic marker clear owns and revalidates the task lock and writes +ledger state before removal; manual clears rearm with a task audit event. +Acceptance coverage now includes real queue restart recovery, production-shaped +fake CLIs, and readiness from an extracted gem. + +**Pages:** [[modules/daemon]], [[commands/doctor]], [[commands/markers]], +[[modules/markers]], [[modules/events]], [[state-model]], [[operating]], +[[testing]]. diff --git a/wiki/modules/config.md b/wiki/modules/config.md index fbb9a3ae..e233a93a 100644 --- a/wiki/modules/config.md +++ b/wiki/modules/config.md @@ -3,7 +3,7 @@ title: Hive::Config type: module source: lib/hive/config.rb created: 2026-04-25 -updated: 2026-06-27 +updated: 2026-07-17 tags: [config, yaml, validation] --- @@ -230,6 +230,7 @@ Runs after merge so a default value can never trigger a failure — only user in 8. **`validate_permissions!`** — top-level, stage-level, review-role, and reviewer-entry `permissions:` specs are parsed by `Hive::PermissionScope` and must be `yolo`, `read-only`, or a valid `scoped` map. Shape errors, unknown presets/keys, and `bash:` plus `tools:` fail during config load; runner capability is checked later when the stage profile is known. `reject_unsupported_permissions_at!` also rejects a block-level `review.adhoc.permissions` key here (put permissions on the individual ad-hoc reviewer entries under `review.adhoc.reviewers` instead). 9. **`validate_babysitter!`** — `babysitter.enabled` and `babysitter.dry_run` must be booleans; `interval` must be integer seconds or a `\d+[smh]` string; `max_concurrent_prs`, `budget_minutes`, and `budget_usd` must be integers >= 1; `labels_ignore` must be an array of strings. 10. **`validate_patrol!`** — `patrol.mode` must be one of `ultrapatrol`, `high`, `medium`, `low`, or `off`; `patrol.enabled`, `patrol.draft_prs`, and `patrol.review_prs` must be booleans when present; `trigger` must be one of the patrol trigger enum values; confidence/severity/count/interval/command shape are validated before the scheduler or `hive patrol` command can run. `patrol.review.reviewers` uses the same reviewer-entry validation as `review.reviewers`, but it is a separate list used only by synthetic `Patrol: ...` review tasks. +11. **`validate_daemon_auto_retry!`** — optional global `daemon.auto_retry` must be a Hash; `daemon.auto_retry.enabled` must be a strict boolean when present. Defaults to `true` and gates only the v1 health-gated auto-retry coordinator (see [[modules/daemon]]); it does not change per-project `daemon.enabled` or `StaleAgentHealer` behavior. Bot attachment capture settings are validated with the other bot numeric keys: `bot.idea_attachment_max_bytes` defaults to 20 MiB and may not diff --git a/wiki/modules/daemon.md b/wiki/modules/daemon.md index fe8dd9cc..0298817a 100644 --- a/wiki/modules/daemon.md +++ b/wiki/modules/daemon.md @@ -3,7 +3,7 @@ title: Hive::Daemon type: module source: lib/hive/daemon/ created: 2026-05-06 -updated: 2026-06-20 +updated: 2026-07-18 tags: [daemon, module, automation, dispatcher] --- @@ -11,7 +11,7 @@ tags: [daemon, module, automation, dispatcher] the auto-advancing dispatcher (ADR-024). Pure logic (`Policy`, `ConcurrencyController`) is separated from I/O (`StatusConsumer`, `ChildSupervisor`, `Logger`, `PrMergeWatcher`, `DigestScheduler`, -`StaleAgentHealer`, `DisplayNameBackfiller`) so +`StaleAgentHealer`, `DisplayNameBackfiller`, `AutoRetry::*`) so the safety-relevant decisions are unit-testable without forking. ## Module map @@ -30,8 +30,9 @@ the safety-relevant decisions are unit-testable without forking. | `Hive::Daemon::DisplayNameBackfiller` | `lib/hive/daemon/display_name_backfiller.rb` | Tick-time self-heal for tasks whose one-shot name generation at `hive new` never landed (agent/codex outage). Re-spawns fire-and-forget `hive generate-name ` for any row whose `Hive::TaskMeta` `display_name` is nil/blank, mirroring `Hive::Commands::New#spawn_name_generator` (detached, pgroup, logged to `/logs/display-name.log`, fully rescued). Anti-churn: an `@inflight` map stores `{pid, at}` per folder, uses `kill(0)` liveness plus `MAX_INFLIGHT_AGE_SEC = 120` to avoid both double-spawns and reused-pid/EPERM pinning, `max_per_tick` (default 2) bounds spawns, and a set name is a natural fixed point. Unexpected row/reap/spawn errors degrade through `:fatal` logging while preserving the no-raise tick contract. Purely additive — never touches markers or dispatch. Logs `display_name_backfill`. | | `Hive::Daemon::TaskIdBackfiller` | `lib/hive/daemon/task_id_backfiller.rb` | Tick-time self-heal for tasks created outside `hive new` (hand-made folder, one `mv`-ed in) whose `meta.yml` has no `id` — `hive new` allocates ids from `Hive::TaskCounter`, so a task that skipped it shows a blank id everywhere (TUI, status, digest, dependency refs). For any row whose `Hive::TaskMeta` `id` is nil it allocates `TaskCounter.next!`, writes it via `TaskMeta.update_id` (every other meta field preserved), and commits the meta on `hive/state` under the per-project commit lock (`Hive::Lock.with_commit_lock`, as every durable committer does) with the per-task `hive_commit(stage_name:, slug:, action: "id-assigned")` call. The `task_id_backfill` event carries `committed:` so a swallowed commit (lock timeout / git error) is visible rather than masquerading as fully durable. Synchronous (no spawn/inflight — assignment is instant), `max_per_tick` (default 5) bounds the per-tick commits, and an assigned id is a natural fixed point. Guards `File.directory?(folder)` first so a row that outlived its folder (e.g. `hive drop` between snapshot and tick) is NOT resurrected by `TaskMeta.write`'s `mkdir_p`. Row/commit errors degrade through `:fatal` / `task_id_backfill_commit_skipped` logging while preserving the no-raise tick contract. Purely additive — never touches markers or dispatch. Logs `task_id_backfill`. | | `Hive::Daemon::PrMergeWatcher` | `lib/hive/daemon/pr_merge_watcher.rb` | Polls `gh pr view --json state` for tasks at 8-finalize/`:complete` and for a narrow set of finalize `ERROR` rows whose PR can still be retired after merge (`git_status_failed`, `claude_launch_failed`). On `MERGED` returns an archive dispatch entry the dispatcher fires. Backs off + drops on persistent gh failures. | +| `Hive::Daemon::AutoRetry::*` | `lib/hive/daemon/auto_retry/` | Health-gated auto-retry coordinator (v1). Classifies only two exact recovery classes (`codex_auth_401`, `claude_launcher`), runs bounded probes, enforces durable 2-attempt budgets under `/daemon_auto_retry.json`, fail-closed stage safety, and queues guarded `markers clear` + same-stage rerun via `DispatchRequestQueue` (`requestor=auto_retry`). Global kill switch: `daemon.auto_retry.enabled` (default true). Does **not** live inside `StaleAgentHealer`. | | `Hive::Daemon::DigestScheduler` | `lib/hive/daemon/digest_scheduler.rb` | Global daily shipped-digest cadence. Persists `last_digested_date` in `/digest_state.json`, applies a first-run no-history guard, computes owed local calendar days after midnight, caps catch-up with `digest.max_catchup_days`, and emits one `hive digest --date D --json` dispatch at a time. | -| `Hive::Daemon::DispatchRequestQueue` | `lib/hive/daemon/dispatch_request_queue.rb` | File-backed queue (`/dispatch_requests/*.json`) of dispatch requests written by producer paths (Telegram bot via `Hive::Bot::DispatchRequestWriter`, hivebox stage-run dispatches, and the 3-plan healer requeue) and consumed by the dispatcher's tick loop. Current wire schema is `hive-dispatch-request.v2`: `requestor` is the closed enum `bot|healer`, and any other `schema_version` is rejected as `unknown_schema_version`. Allowlists state-mutating verbs (`run develop brainstorm plan review open-pr artifacts finalize archive markers`); rejects everything else with a logged `:dispatch_request_rejected` event. The single-dispatcher invariant lives here: producers write, the daemon dispatches. See [[architecture]] §"Single-dispatcher contract". | +| `Hive::Daemon::DispatchRequestQueue` | `lib/hive/daemon/dispatch_request_queue.rb` | File-backed queue (`/dispatch_requests/*.json`) of dispatch requests written by Telegram, hivebox, the 3-plan healer, and health-gated auto-retry. Current `hive-dispatch-request.v2` uses the closed `requestor` enum `bot|healer|auto_retry`; automatic requests also carry `recovery_id` + `recovery_step`. Unknown schema versions and non-allowlisted verbs are rejected. See [[architecture]] §"Single-dispatcher contract". | | `Hive::Daemon::QueueDirectory` | `lib/hive/daemon/queue_directory.rb` | Shared `directory_for(dirname:, state_home:)` helper used by both dispatch queues so the owner-only (0700) per-queue directory invariant — the de-facto auth boundary for the dispatch channel — lives in one place (#253). | | `Hive::Commands::Daemon` | `lib/hive/commands/daemon.rb` | Thor subcommand surface (`start` / `stop` / `status` / `reload` / `tail` / `install` / `enable` / `disable` / `queue`). Owns PID/signal lifecycle, service installation, per-project enrollment, and read-only dispatch-request queue inspection. `queue` delegates to `Hive::Commands::Daemon::QueueCommand`. | | `Hive::Commands::Daemon::QueueCommand` | `lib/hive/commands/daemon/queue_command.rb` | Extracted read-only queue-inspection surface (`hive daemon queue list/show/prune`) — touches only `queue_args`/`json`/`hive_home`, orthogonal to the daemon lifecycle, mirroring the `ServiceInstaller` extraction (#254). Internal IO/parse failures are wrapped in `Hive::InternalError` (exit 70). | @@ -66,7 +67,8 @@ cadence for changes the cheap probe cannot see. Each full tick runs in order: reap completed children -> enforce child timeouts -> prune dispatch-result notices -> **tick the digest scheduler** -> fetch status -> heal stale agent markers -> backfill missing display names -> -backfill missing meta ids -> tick the PR-merge watcher -> **process dispatch requests** -> patrol dispatches +backfill missing meta ids -> tick the PR-merge watcher -> **auto-retry coordinator** +-> **process dispatch requests** -> patrol dispatches -> per-row dispatch -> prune baselines -> refresh cheap-probe mtime fingerprints. During per-row dispatch, whitelisted `8-finalize` `ERROR` rows (`git_status_failed`, `claude_launch_failed`) are enqueued into the @@ -74,12 +76,52 @@ merge watcher before the generic policy table skips `error` rows. Because the watcher tick already ran earlier in the same full tick, a newly enqueued row is polled on a later tick; the watcher emits an archive command with `--recover-merged-error-reason` only after GitHub reports the -PR as `MERGED`. +PR as `MERGED`. Auto-retry runs **after** the merge watcher (finalize merge +recovery keeps precedence) and **before** dispatch-request processing so a +newly queued clear can pass normal queue gates in the same tick. Dispatch requests come BEFORE the row-scan so a slug whose request just dispatched this tick is already in-flight in the controller and the row scan's per-slug in-flight gate (`controller.running_task?`) keeps the same tick from double-spawning. +### Health-gated auto-retry (v1) + +Only two terminal `ERROR` shapes are eligible: + +| Recovery class | Marker | Extra evidence | +|---|---|---| +| `codex_auth_401` | `ERROR reason=implementer_failed` on `4-execute` with effective execute agent `codex` | Current-marker-scoped diagnostics contain HTTP 401 plus missing bearer/basic-auth wording | +| `claude_launcher` | `ERROR reason=claude_launch_failed` with effective stage agent `claude` and `claude.mode: tmux` | Fixed launcher/packaging/readiness signature; session-name collisions, quota walls, and permission prompts stay manual | + +Every other marker stays parked. Before clearing, the coordinator proves +replay safety (clean execute worktree; blank/unanswered brainstorm; blank +plan only) twice — once before probes and again inside automatic +`hive markers clear --recovery-id …`. Durable attempt accounting keys by +project + task id/slug + stage + reason (not marker id): max 2 dispatched +retries, second attempt requires a different healthy fingerprint and a +30-minute backoff. A successful **manual** clear rearms the budget; an +automatic clear carries `recovery_id` so it cannot rearm itself. Audit lands +on both daemon log (`auto_retry_decision`) and task `events.jsonl`. + +Health is stronger than a zero exit. Both recovery classes reuse Doctor's +required-agent API. Codex then runs bounded `login status` and a +production-shaped JSONL smoke (`exec --json ... -`, prompt on stdin) and +requires a successful completion record. Claude verifies the packaged wrapper, +the production readiness detector, bounded tmux/version probes for the +configured runtime, and a fresh `hive version --json` +version/fingerprint/binary-path match. Unhealthy results are reused until a +cheap executable/config/credential signal changes or the 30-minute fallback +cadence expires. + +Clear/rerun publication and recovery are crash-safe: the continuation sidecar +is durable before the runnable clear, every automatic request must match a +coherent ledger record, and deterministic request IDs make reconciliation +idempotent. Restart reconciliation handles a still-present marker and the +markerless post-clear window, and maps dispatched attempts to `failed`, +`succeeded`, `aborted`, or `exhausted` from authoritative status. Corrupt +ledger structure suspends automatic writes. Disabling the global kill switch +also makes the dispatcher reject pending automatic requests. + Digest dispatches happen before status fetch because they are global, not project-row driven. The dispatcher tracks them with synthetic project/stage `digest/digest`; when the child is reaped, the scheduler advances its cursor diff --git a/wiki/modules/events.md b/wiki/modules/events.md index 841b623e..2ff865ea 100644 --- a/wiki/modules/events.md +++ b/wiki/modules/events.md @@ -3,11 +3,11 @@ title: Hive::Events type: module source: lib/hive/events.rb created: 2026-05-23 -updated: 2026-05-23 +updated: 2026-07-18 tags: [module, events, observability, status, append-only] --- -**TLDR**: Append-only task-local event log + derived `status.md` renderer. Every stage run is bracketed by `stage_enter` / `stage_exit`, every agent spawn by `agent_start` / `agent_end`, and the review loop adds per-phase `agent_start` / `agent_end` pairs on top. Records are written one O_APPEND JSON line at a time to `/events.jsonl`; `status.md` is rewritten via atomic rename after every emit and tails the last 20 events plus the currently-open agent. +**TLDR**: Append-only task-local event log + derived `status.md` renderer. Every stage run is bracketed by `stage_enter` / `stage_exit`, every agent spawn by `agent_start` / `agent_end`, and the review loop adds per-phase `agent_start` / `agent_end` pairs on top. Records are written one O_APPEND JSON line at a time to `/events.jsonl`; `status.md` is rewritten via atomic rename after every emit and tails the last 20 events plus the currently-open agent. Optional bounded `details` objects support auto-retry audit without breaking old readers. ## Event types (`Hive::Events::EVENT_TYPES`) @@ -20,6 +20,7 @@ tags: [module, events, observability, status, append-only] | `error` | `Stages::Base.with_stage_events` rescue path; `emit_marker_event` for error markers | Stage raised, or marker landed on `:error` / `:review_error` / `:review_ci_stale` / `:review_stale` | | `round_waiting` | `Stages::Base.emit_marker_event` | Brainstorm or plan stage closed with `:waiting` marker | | `round_complete` | same | Brainstorm or plan stage closed with `:complete` marker | +| `auto_retry_decision` | `Hive::Daemon::AutoRetry::Coordinator`, manual `hive markers clear` | Positive/negative/terminal automatic decisions (throttled negatives), or `manual_rearm`; `details` carries the decision plus bounded probe status, timeout, duration, exit status, and redacted excerpts | `ROUND_EVENT_STAGES = %w[brainstorm plan]` is the registry that gates round events — adding a new stage that publishes `:waiting` / `:complete` round markers requires extending this list so `emit_marker_event` stays in sync with the producers. diff --git a/wiki/modules/markers.md b/wiki/modules/markers.md index fc8e0a32..d60c69c2 100644 --- a/wiki/modules/markers.md +++ b/wiki/modules/markers.md @@ -3,7 +3,7 @@ title: Hive::Markers type: module source: lib/hive/markers.rb created: 2026-04-25 -updated: 2026-06-18 +updated: 2026-07-18 tags: [marker, protocol, flock] --- @@ -96,6 +96,11 @@ Parses the attribute string into a Hash. Format: `key=value` pairs, optional dou - `Stages::Execute#run_pass` writes `EXECUTE_WAITING` / `EXECUTE_COMPLETE` after validating final output, branch ancestry, worktree cleanliness, and research-mode eligibility. - `Stages::Review.run!` writes `REVIEW_WORKING` at every phase entry; the orchestrator owns every terminal `REVIEW_*` marker per ADR-005's last-marker-wins rule. - `Hive::Commands::Status` reads markers to render the table. +- `Hive::Commands::Markers` clears recovery markers under `.markers-lock`. + Health-gated automatic clears additionally own the task `.lock`, validate a + durable recovery ID and global kill switch, and persist retry accounting + before marker removal; manual clears rearm the budget and emit a + `manual_rearm` audit event. See [[commands/markers]] and [[modules/daemon]]. ## Backlinks diff --git a/wiki/operating.md b/wiki/operating.md index 2b24d46a..4492ec74 100644 --- a/wiki/operating.md +++ b/wiki/operating.md @@ -3,7 +3,7 @@ title: Operating Hive type: operating source: README.md, bin/hv, install.sh, lib/hive/commands/daemon.rb, lib/hive/commands/babysit.rb, lib/hive/commands/bot.rb, examples/systemd/, examples/launchd/, openclaw/skills/hive/SKILL.md, openclaw/README.md created: 2026-05-07 -updated: 2026-06-25 +updated: 2026-07-18 tags: [operating, daemon, bot, systemd, launchd, install] --- @@ -509,6 +509,7 @@ drain path when you are not using systemd/launchd. | Status + uptime | `hive daemon status` (`--json` for envelope) | | Follow the structured log | `hive daemon tail` | | Reload caps without restart | edit `~/.config/hive/config.yml` → `hive daemon reload` | +| Stop health-gated auto-retry (including queued work) | set `daemon.auto_retry.enabled: false` globally → `hive daemon reload` | | Disable a project mid-flight | `hive daemon disable PROJECT` → `hive daemon reload`* | | Enable a project mid-flight | `hive daemon enable PROJECT` → `hive daemon reload`* | | Take over a stuck task manually | `hive tui`, focus the row, press `s` | @@ -565,6 +566,21 @@ daemon: `hive daemon reload` (or restart) picks up the new values. +Health-gated recovery is default-on and separately kill-switchable: + +```yaml +daemon: + auto_retry: + enabled: false +``` + +After reload, the coordinator stops creating work and the dispatcher rejects +already-pending `requestor=auto_retry` clear/rerun requests. An automatic +clear also rereads this global setting immediately before marker removal. +Manual marker recovery and the older stale-agent healer are separate paths. +Re-enable only after `hive doctor` is green and the affected CLI login/runtime +is healthy. + ## Cost-runaway response If `daemon.log` shows unexpected dispatch volume: @@ -615,6 +631,17 @@ edit didn't land where the daemon reads from. Check `daemon: { enabled: false }` — `hive daemon disable PROJECT` is the safest path. +**An eligible auth/launcher error remains parked.** Inspect +`hive daemon tail` for `auto_retry_decision`. Common refusal rationales are a +stale/unscoped diagnostic, substantive brainstorm/plan feedback, dirty +worktree, live task lock, failed Doctor row, malformed Codex completion, +configured Claude runtime failure, or stale daemon/CLI identity. Probe output +is redacted and bounded in both daemon and task audit events. Unhealthy probes +are intentionally retried only when a cheap health signal changes or after 30 +minutes. Corrupt `daemon_auto_retry.json` suspends automatic writes; preserve +the file for diagnosis and recover manually instead of deleting it to reset +the retry budget. + ## Backlinks - [[commands/daemon]] · [[modules/daemon]] diff --git a/wiki/state-model.md b/wiki/state-model.md index 4547c01b..e4fbdce5 100644 --- a/wiki/state-model.md +++ b/wiki/state-model.md @@ -3,11 +3,11 @@ title: State Model type: data-model source: lib/hive/task.rb, lib/hive/markers.rb, lib/hive/config.rb, lib/hive/lock.rb, lib/hive/worktree.rb, lib/hive/metrics.rb, lib/hive/usage_db.rb, lib/hive/bot/*, lib/hive/patrol/review_handoff.rb, lib/hive/commands/adhoc_review.rb, lib/hive/daemon/display_name_backfiller.rb, lib/hive/daemon/dispatch_request_queue.rb, lib/hive/web/status_feed.rb, web/app/models/status_broadcaster.rb created: 2026-04-25 -updated: 2026-06-27 +updated: 2026-07-18 tags: [state, filesystem, model, architecture, review, task-id, display-name, archive, web] --- -**TLDR**: Hive's workflow state has no application database. Persistent task/project state lives in two filesystem trees per project — `/.hive-state/` (an orphan-branch worktree holding task folders, configs, locks, logs) and `~/Dev/.worktrees//` (feature worktrees holding actual code) — plus one global `~/.config/hive/config.yml` (or `HIVE_HOME/config.yml` / a migrated legacy registry). Token-usage metrics are the exception and use the SQLite store described in [[token-usage]]. Hivebox web adds no workflow tables: it reads `hive status` snapshots through `StatusFeed`/`StatusBroadcaster` and writes daemon dispatch requests as JSON files under the global state home. The workflow "data model" is the directory layout, marker grammar, YAML sidecars, and runtime JSON queue files described below. +**TLDR**: Hive's workflow state has no application database. Persistent task/project state lives in two filesystem trees per project — `/.hive-state/` (an orphan-branch worktree holding task folders, configs, locks, logs) and `~/Dev/.worktrees//` (feature worktrees holding actual code) — plus one global `~/.config/hive/config.yml` (or `HIVE_HOME/config.yml` / a migrated legacy registry) and daemon ledger files under the global state home (including `daemon_auto_retry.json` for health-gated retry budgets). Token-usage metrics are the exception and use the SQLite store described in [[token-usage]]. Hivebox web adds no workflow tables: it reads `hive status` snapshots through `StatusFeed`/`StatusBroadcaster` and writes daemon dispatch requests as JSON files under the global state home. The workflow "data model" is the directory layout, marker grammar, YAML sidecars, and runtime JSON queue files described below. ## Stage directory layout @@ -147,10 +147,12 @@ created_at: project: slug: argv: ["hive", "", ...] -requestor: bot|healer +requestor: bot|healer|auto_retry chat_id: update_id: trigger: +recovery_id: +recovery_step: clear|rerun ``` The current strict wire contract is `hive-dispatch-request.v2`: v2 adds the @@ -161,6 +163,13 @@ closed `requestor: healer` producer used by `StaleAgentHealer` while preserving `unknown_schema_version`; older schema files remain in `schemas/` for pinned validators, not for mixed-version live queue operation. +`auto_retry` requests execute only when recovery ID, step, project, slug, +stage, reason, and durable phase agree with `daemon_auto_retry.json`. The +dispatcher rejects and removes incoherent authorization or any pending +automatic request observed while the global kill switch is disabled. +Automatic request IDs are deterministic per recovery/step, so replaying a +publication or continuation cannot spend the attempt budget twice. + `Hive::Daemon::DispatchRequestQueue.valid_argv?` requires `argv[0] == "hive"` and allowlists only workflow-mutating verbs (`run`, `develop`, `brainstorm`, `plan`, `review`, `open-pr`, `artifacts`, `finalize`, `archive`, `markers`). diff --git a/wiki/testing.md b/wiki/testing.md index 3ee77fea..70fc9105 100644 --- a/wiki/testing.md +++ b/wiki/testing.md @@ -3,7 +3,7 @@ title: Testing type: reference source: test/, Rakefile, bin/hive-eval, .rubocop.yml, .github/workflows/ci.yml, .github/workflows/release.yml, config/brakeman.ignore created: 2026-04-25 -updated: 2026-06-25 +updated: 2026-07-18 tags: [test, minitest, fixtures] --- @@ -92,6 +92,7 @@ task default: :test | `screenote_oauth_live_test.rb`, `screenote_capture_live_test.rb` | Opt-in live Screenote tests — real OAuth discovery, rate-limited dynamic registration when enabled, auth-code token exchange when preseeded, and the blocked real `create_screenshot_upload` round-trip through Screenote's non-interactive test-token endpoint once that endpoint ships. | | `daemon/status_consumer_test.rb` | `Hive::Daemon::StatusConsumer` — `hive status --json` envelope parsing, schema-version skew handling, strict `live_task_lock` coercion, legacy project filtering, and local `state_file` mtime re-stat so daemon edit-resume decisions keep subsecond precision even though public JSON timestamps are whole-second ISO8601. | | `daemon/stale_agent_healer_test.rb` | `Hive::Daemon::StaleAgentHealer` — stale `AGENT_WORKING` healing, wedged `REVIEW_WORKING` lock cleanup, and bounded daemon auto-recovery for `review_agent_died`, reviewer partial failures caused only by Claude/tmux expected-output session death, fix-phase `fix_failed` only when the message is the known Claude stop-hook completion failure, `8-finalize` `ERROR reason=unpushed_commits`, elapsed `limits_reached` cooldown markers (including terminal `ERROR reason=limits_reached` on `4-execute`), and non-review terminal agent-loss errors (`2-brainstorm`, `3-plan`, `4-execute`, `7-artifacts`, `8-finalize` `ERROR reason=tmux_session_terminated` or `reason=agent_orphaned`). Terminal-error coverage pins marker-id guarded clears, live-lock skips, manual repository-state skips, shared budgets across fresh marker ids, per-task budget isolation, pre-clear dispatch-baseline seeding, the `3-plan` `hive plan ... --from 3-plan` dispatch-request requeue / `heal_requeued` trace for both agent-loss and `limits_reached` clears, the distinct `heal_requeue_failed` event when enqueueing fails after a successful clear, one-shot `marker_heal_exhausted` logging, and the load-bearing AgentLimit wire-message → held review marker → cooldown-boundary assertion. | +| `daemon/auto_retry/*_test.rb`, `daemon/dispatcher_test.rb`, `integration/daemon_auto_retry_test.rb`, `integration/gem_package_scripts_test.rb` | Health-gated auto-retry v1 — exact/current-episode classification, production-shaped fake CLI probes with concurrent bounded capture, shared Doctor gate, configured Claude runtime/readiness, strict CLI identity, cadence/fair budgeting/exhaustion, corrupt-state suspension, late-lock safety, durable authorization/accounting, pending-work kill switch, real-queue restart reconciliation, dual-surface audit, and extracted-gem readiness. | | `hv_test.rb` | `bin/hv` — refuses unsafe Apache Hive fallback paths (`/usr/bin/hive`, `/opt/hive/bin/hive`) and verifies `HIVE_BIN_OVERRIDE` can point at a custom Hive CLI install path. | | `gemspec_test.rb`, `install_script_test.rb` | RubyGem/install packaging — `hv` stays out of `spec.executables` so RubyGems does not create a broken Ruby binstub for the bash launcher; the bash installer writes its own `hv` wrapper and does not expect a gem-installed `hv` shim. | | `babysitter/dry_run_env_test.rb` | `Hive::Babysitter::DryRunEnv` plus `bin/hive-babysitter-stub-git` / `bin/hive-babysitter-stub-gh` — PATH overlay wrapper handoff, command-local `RUBYOPT`/`RUBYLIB` startup injection scrubbing before stub handoff, relative PATH real-binary canonicalization, non-absolute `HIVE_BABYSITTER_REAL_*` refusal, command-local `HIVE_BABYSITTER_REAL_*` and `HIVE_BABYSITTER_DRY_RUN_LOG` override resistance, recording fake binaries pinned to the current test runner Ruby so git-stub PATH pinning cannot switch fixture interpreters, default-deny skips, read-only passthrough, argv-wide and positional `gh` host-override skips, `GH_HOST` / `GH_REPO` / enterprise-token env scrubbing, `gh` config env scrubbing plus fresh empty `HOME`/`GH_CONFIG_DIR` passthrough roots, `gh api` implicit-POST payload flag blocking plus explicit-GET file/cache guards, host-default non-token `gh auth status` passthrough with token-display and hostname skips, browser-launch flag skips plus `w` inside value-taking `gh` read-option values, git executable/write-option skips, `remote show` without `-n` skipping before repo-configured transport helpers can run, exact read-only `git branch` forms and mixed branch mutation skips, env config/command seam skips including `GIT_EXEC_PATH`, `GIT_ASKPASS`, and `SSH_ASKPASS`, hermetic HOME/XDG/local git config passthrough guards, `--textconv` abbreviation, `cat-file --filters`, and git signature-verification skips, subcommand `-p` passthrough, `grep`/`ls-files` read-option exceptions, grep pager `--open-files-in-pager` abbreviations and `-O` forms including clustered `-nO`, value-taking grep short options such as `-eTODO` / `-fNEEDLEFILE.txt`, consumed `grep -e --` separator handling before pager/textconv guards, pathspec separator handling, symlinked skip-log refusal, FIFO skip-log refusal through a timeout-bounded stub capture, and ASCII control-character escaping in skip logs/stderr, and invalid/non-UTF-8 `git`/`gh` argv skip-log regressions. |