generate-formula-api: attach vulnerabilities from advisory-database

Library/Homebrew/dev-cmd/generate-formula-api.rb

@@ -6,6 +6,7 @@
 require "executables_db"
 require "fileutils"
 require "formula"
+require "vulns/advisory_database"
 
 module Homebrew
   module DevCmd
@@ -49,6 +50,7 @@ def run
           odie "Failed to download #{executables_path}"
         end
         executables = ExecutablesDB.new(executables_path.to_s).to_hash
+        advisories = load_advisory_database
 
         Homebrew.with_no_api_env do
           tap_migrations_json = JSON.dump(tap.tap_migrations)
@@ -65,6 +67,9 @@ def run
               name = formula.name
               all_formulae[name] = formula.to_hash_with_variations
               all_formulae[name]["executables"] = executables[name] if executables.key?(name)
+              if (vulns = advisories&.status_for(name, formula.pkg_version))
+                all_formulae[name]["vulnerabilities"] = vulns
+              end
               json = JSON.pretty_generate(all_formulae[name])
               html_template_name = html_template(name)
 
@@ -104,6 +109,16 @@ def run
 
       private
 
+      # An advisory-database or network failure must not break the API build;
+      # the `vulnerabilities` field is omitted and the build proceeds.
+      sig { returns(T.nilable(Homebrew::Vulns::AdvisoryDatabase)) }
+      def load_advisory_database
+        Homebrew::Vulns::AdvisoryDatabase.load
+      rescue Homebrew::Vulns::CachedFeed::Error, ErrorDuringExecution => e
+        opoo "Skipping vulnerabilities field: #{e.message.lines.first&.strip}"
+        nil
+      end
+
       sig { params(title: String).returns(String) }
       def html_template(title)
         <<~EOS

Library/Homebrew/test/dev-cmd/generate-formula-api_spec.rb

@@ -5,41 +5,78 @@
 require "dev-cmd/generate-formula-api"
 
 RSpec.describe Homebrew::DevCmd::GenerateFormulaApi do
-  it_behaves_like "parseable arguments"
-
-  it "writes formula executables to generated formula data" do
+  before do
     core_tap = instance_double(CoreTap, installed?: true, name: "homebrew/core", formula_names: ["foo"],
                                         alias_table: {}, formula_renames: {}, git_head: "formula-head",
                                         tap_migrations: {})
-    bottle_tag = Utils::Bottles::Tag.from_symbol(:arm64_sonoma)
-
     allow(CoreTap).to receive(:instance).and_return(core_tap)
     allow(Formulary).to receive(:enable_factory_cache!)
     allow(Formula).to receive(:generating_hash!)
     allow(Formulary).to receive(:factory).with("foo").and_return(
-      instance_double(Formula, name: "foo", to_hash_with_variations: { "name" => "foo" }),
+      instance_double(Formula, name: "foo", pkg_version: PkgVersion.parse("1.0.0"),
+                               to_hash_with_variations: { "name" => "foo" }),
     )
     allow(Homebrew::API).to receive(:download_executables_file_from_github_packages!) do |target|
       target.write "foo(1.0.0):foo-tool food\n"
       true
     end
     allow(Homebrew::API::Formula::FormulaStructGenerator).to receive(:generate_formula_struct_hash)
-      .with({ "name" => "foo", "executables" => ["foo-tool", "food"] }, bottle_tag:)
-      .and_return(
-        instance_double(
-          Homebrew::API::FormulaStruct,
-          serialize: { "name" => "foo", "executables" => ["foo-tool", "food"] },
-        ),
-      )
-    stub_const("OnSystem::VALID_OS_ARCH_TAGS", [bottle_tag])
+      .and_return(instance_double(Homebrew::API::FormulaStruct, serialize: { "name" => "foo" }))
+    stub_const("OnSystem::VALID_OS_ARCH_TAGS", [Utils::Bottles::Tag.from_symbol(:arm64_sonoma)])
+  end
+
+  it_behaves_like "parseable arguments"
+
+  it "writes formula executables to generated formula data" do
+    allow(Homebrew::Vulns::AdvisoryDatabase).to receive(:load).and_return(nil)
 
     Dir.mktmpdir do |tmpdir|
       path = Pathname.new(tmpdir)
       path.cd { described_class.new([]).run }
 
-      expect(JSON.parse((path/"_data/formula/foo.json").read)["executables"]).to eq(["foo-tool", "food"])
-      expect(JSON.parse((path/"api/internal/formula.arm64_sonoma.json").read)
-        .dig("formulae", "foo", "executables")).to eq(["foo-tool", "food"])
+      data = JSON.parse((path/"_data/formula/foo.json").read)
+      expect(data["executables"]).to eq(["foo-tool", "food"])
+      expect(data).not_to have_key("vulnerabilities")
+    end
+  end
+
+  it "attaches vulnerabilities from the advisory-database corpus to the public formula JSON" do
+    advisories = Homebrew::Vulns::AdvisoryDatabase.new({
+      "meta"       => {},
+      "advisories" => {
+        "foo" => [{
+          "id"       => "BREW-foo-CVE-2024-1234",
+          "upstream" => ["CVE-2024-1234"],
+          "affected" => [{
+            "package"            => { "ecosystem" => "Homebrew", "name" => "foo" },
+            "ranges"             => [{ "type" => "ECOSYSTEM", "events" => [{ "introduced" => "0" }] }],
+            "ecosystem_specific" => { "fix" => nil },
+          }],
+        }],
+      },
+    })
+    allow(Homebrew::Vulns::AdvisoryDatabase).to receive(:load).and_return(advisories)
+
+    Dir.mktmpdir do |tmpdir|
+      path = Pathname.new(tmpdir)
+      path.cd { described_class.new([]).run }
+
+      vulns = JSON.parse((path/"_data/formula/foo.json").read)["vulnerabilities"]
+      expect(vulns["open"].map { |e| e["id"] }).to eq ["BREW-foo-CVE-2024-1234"]
+      expect(vulns["patched"]).to eq []
+      expect(vulns["fixed_count"]).to eq 0
+    end
+  end
+
+  it "omits the vulnerabilities field and warns when the advisory feed cannot be loaded" do
+    allow(Homebrew::Vulns::AdvisoryDatabase).to receive(:load)
+      .and_raise(Homebrew::Vulns::CachedFeed::Error, "boom")
+
+    Dir.mktmpdir do |tmpdir|
+      path = Pathname.new(tmpdir)
+      expect { path.cd { described_class.new([]).run } }
+        .to output(/Skipping vulnerabilities field: boom/).to_stderr
+      expect(JSON.parse((path/"_data/formula/foo.json").read)).not_to have_key("vulnerabilities")
     end
   end
 end

Library/Homebrew/test/vulns/advisory_database_spec.rb

@@ -0,0 +1,128 @@
+# typed: true
+# frozen_string_literal: true
+
+require "vulns/advisory_database"
+
+RSpec.describe Homebrew::Vulns::AdvisoryDatabase do
+  def record(id, formula, events:, fix: nil, upstream: nil, summary: nil, severity: nil)
+    {
+      "id"       => id,
+      "upstream" => upstream,
+      "summary"  => summary,
+      "severity" => severity,
+      "affected" => [{
+        "package"            => { "ecosystem" => "Homebrew", "name" => formula },
+        "ranges"             => [{ "type" => "ECOSYSTEM", "events" => events }],
+        "ecosystem_specific" => { "fix" => fix },
+      }],
+    }.compact
+  end
+
+  def db(by_formula)
+    described_class.new({ "meta"       => { "count" => by_formula.each_value.sum(&:size) },
+                          "advisories" => by_formula })
+  end
+
+  describe "#initialize" do
+    it "raises Error when the top-level value is not a JSON object" do
+      expect { described_class.new([]) }
+        .to raise_error(Homebrew::Vulns::CachedFeed::Error, /not a JSON object/)
+    end
+
+    it "raises Error when the advisories key is missing" do
+      expect { described_class.new({ "meta" => {} }) }
+        .to raise_error(Homebrew::Vulns::CachedFeed::Error, /no 'advisories' key/)
+    end
+
+    it "distinguishes a wrong-type advisories value from a missing key" do
+      expect { described_class.new({ "advisories" => [] }) }
+        .to raise_error(Homebrew::Vulns::CachedFeed::Error, /'advisories' is not a JSON object/)
+    end
+  end
+
+  describe "#records_for" do
+    it "wraps each record for the formula in a Vulnerability" do
+      d = db({ "unzip" => [record("BREW-unzip-CVE-1", "unzip",
+                                  events: [{ "introduced" => "0" }, { "fixed" => "6.0_29" }])] })
+      expect(d.records_for("unzip").map(&:id)).to eq ["BREW-unzip-CVE-1"]
+      expect(d.records_for("unzip").first).to be_a Homebrew::Vulns::Vulnerability
+    end
+
+    it "returns [] for a formula with no records" do
+      expect(db({}).records_for("nope")).to eq []
+    end
+  end
+
+  describe "#status_for" do
+    let(:corpus) do
+      db({
+        "unzip" => [
+          record("BREW-unzip-CVE-2014-8139", "unzip",
+                 events: [{ "introduced" => "0" }, { "fixed" => "6.0_29" }],
+                 fix: "patch", upstream: ["CVE-2014-8139"], summary: "s",
+                 severity: [{ "type"  => "CVSS_V3",
+                              "score" => "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" }]),
+          record("BREW-unzip-CVE-2021-4217", "unzip",
+                 events:   [{ "introduced" => "0" }],
+                 upstream: ["CVE-2021-4217"]),
+          record("BREW-unzip-CVE-2016-0001", "unzip",
+                 events: [{ "introduced" => "0" }, { "fixed" => "6.0_10" }],
+                 fix: "bump", upstream: ["CVE-2016-0001"]),
+        ],
+      })
+    end
+
+    it "partitions into open (still in range), patched (fix: patch, past range), and fixed_count" do
+      status = corpus.status_for("unzip", "6.0_29")
+
+      expect(status["open"].map { |e| e["id"] }).to eq ["BREW-unzip-CVE-2021-4217"]
+      expect(status["open"].first["upstream"]).to eq ["CVE-2021-4217"]
+      expect(status["patched"].map { |e| e["id"] }).to eq ["BREW-unzip-CVE-2014-8139"]
+      expect(status["patched"].first["severity"]).to eq "critical"
+      expect(status["patched"].first["fixed_in"]).to eq "6.0_29"
+      expect(status["fixed_count"]).to eq 1
+    end
+
+    it "counts a patch record whose fixed boundary is above pkg_version as open" do
+      status = corpus.status_for("unzip", "6.0_28")
+      expect(status["open"].map { |e| e["id"] })
+        .to contain_exactly("BREW-unzip-CVE-2014-8139", "BREW-unzip-CVE-2021-4217")
+      expect(status["patched"]).to eq []
+    end
+
+    it "returns nil when the corpus has no records for the formula" do
+      expect(corpus.status_for("nope", "1.0")).to be_nil
+    end
+
+    it "ignores :not_applicable records rather than counting them as fixed or patched" do
+      d = db({ "foo" => [
+        record("BREW-foo-CVE-1", "foo",
+               events: [{ "introduced" => "2.0" }, { "fixed" => "3.0" }], fix: "bump"),
+        record("BREW-foo-CVE-2", "foo",
+               events: [{ "introduced" => "2.0" }, { "fixed" => "3.0" }], fix: "patch"),
+      ] })
+      status = d.status_for("foo", "1.0")
+      expect(status["open"]).to eq []
+      expect(status["patched"]).to eq []
+      expect(status["fixed_count"]).to eq 0
+    end
+
+    it "compacts nil fields out of each entry hash" do
+      status = corpus.status_for("unzip", "6.0_29")
+      expect(status["open"].first.keys).to eq %w[id upstream]
+    end
+
+    it "accepts a PkgVersion" do
+      require "pkg_version"
+      expect(corpus.status_for("unzip", PkgVersion.parse("6.0_29"))["fixed_count"]).to eq 1
+    end
+  end
+
+  describe "#formulae and #meta" do
+    it "exposes the index keys and meta block" do
+      d = db({ "a" => [], "b" => [] })
+      expect(d.formulae).to contain_exactly("a", "b")
+      expect(d.meta["count"]).to eq 0
+    end
+  end
+end

Library/Homebrew/vulns/advisory_database.rb

@@ -0,0 +1,115 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "vulns/cached_feed"
+require "vulns/vulnerability"
+
+module Homebrew
+  module Vulns
+    # Reader for the concatenated `BREW-*` OSV corpus published by
+    # Homebrew/advisory-database at `data/advisories.json` (built by that
+    # repository's `AdvisoryIndex` via `rake advisories:concat`).
+    #
+    # Consumed by `brew generate-formula-api` to attach a `vulnerabilities`
+    # field to each formula's API JSON, and by `brew vulns` (Phase 4) as the
+    # local `ecosystem: Homebrew` range source until osv.dev ingests the feed.
+    class AdvisoryDatabase < CachedFeed
+      DATA_URL = "https://raw.githubusercontent.com/Homebrew/advisory-database/" \
+                 "main/data/advisories.json"
+
+      sig { override.returns(String) }
+      def self.data_url = DATA_URL
+
+      sig { override.returns(String) }
+      def self.cache_filename = "advisories.json"
+
+      sig { override.params(data: T.anything).void }
+      def initialize(data)
+        super
+        raise Error, "advisory index is not a JSON object" unless (top = as_hash(data))
+        raise Error, "advisory index has no 'advisories' key" unless top.key?("advisories")
+        unless (advisories = as_hash(top["advisories"]))
+          raise Error, "advisory index 'advisories' is not a JSON object"
+        end
+
+        @advisories = T.let(advisories, T::Hash[String, T.untyped])
+        @meta = T.let(as_hash(top["meta"]) || {}, T::Hash[String, T.untyped])
+      end
+
+      sig { returns(T::Hash[String, T.untyped]) }
+      attr_reader :meta
+
+      sig { returns(T::Array[String]) }
+      def formulae
+        @advisories.keys
+      end
+
+      # {Vulnerability} wrappers for every `BREW-*` record whose
+      # `affected[0].package.name` is `formula_name`.
+      sig { params(formula_name: String).returns(T::Array[Vulnerability]) }
+      def records_for(formula_name)
+        Array(@advisories[formula_name]).filter_map do |record|
+          Vulnerability.new(record) if record.is_a?(Hash)
+        end
+      end
+
+      Entry = Struct.new(:id, :upstream, :summary, :severity, :fix, :fixed_in, keyword_init: true) do
+        sig { returns(T::Hash[String, T.untyped]) }
+        def to_api_hash
+          to_h.transform_keys(&:to_s).compact
+        end
+      end
+
+      # Evaluate every record for `formula_name` against `pkg_version` and
+      # return the `{open:, patched:}` shape used by the formula API JSON and
+      # `brew info`. `open` are records whose `ECOSYSTEM` range still contains
+      # `pkg_version`; `patched` are records where `ecosystem_specific.fix` is
+      # `"patch"` (Homebrew ships a `resolves`-annotated patch); `fixed_count`
+      # counts bump-fixed records that no longer apply. Returns `nil` when the
+      # corpus has no records for the formula so callers can distinguish
+      # "checked, clean" from "not covered".
+      sig {
+        params(formula_name: String, pkg_version: T.any(String, PkgVersion))
+          .returns(T.nilable(T::Hash[String, T.untyped]))
+      }
+      def status_for(formula_name, pkg_version)
+        records = records_for(formula_name)
+        return if records.empty?
+
+        version = pkg_version.to_s
+        open = T.let([], T::Array[Entry])
+        patched = T.let([], T::Array[Entry])
+        fixed_count = 0
+
+        records.each do |vuln|
+          eco = vuln.affected.first&.dig("ecosystem_specific") || {}
+          status = vuln.range_status("Homebrew", formula_name, version)
+          entry = Entry.new(
+            id:       vuln.id,
+            upstream: vuln.upstream.presence || vuln.aliases.presence,
+            summary:  vuln.summary,
+            severity: vuln.severity&.to_s,
+            fix:      eco["fix"],
+            fixed_in: status&.fixed_in,
+          ).freeze
+          case status&.state
+          when nil, :affected then open << entry
+          when :fixed
+            if eco["fix"] == "patch"
+              patched << entry
+            else
+              fixed_count += 1
+            end
+          when :not_applicable then next
+          end
+        end
+
+        {
+          "open"        => open.sort_by(&:id).map(&:to_api_hash),
+          "patched"     => patched.sort_by(&:id).map(&:to_api_hash),
+          "fixed_count" => fixed_count,
+        }
+      end
+    end
+  end
+end