Fix fetching of SAML metadata for large aggregate endpoints

modules/auth_saml/app/contracts/saml/providers/base_contract.rb

@@ -47,6 +47,8 @@ def self.model
                 url: { allow_blank: true, allow_nil: true, schemes: %w[http https] },
                 if: -> { model.metadata_url_changed? }
 
+      attribute :idp_entity_id
+
       attribute :idp_sso_service_url
       validates :idp_sso_service_url,
                 url: { schemes: %w[http https] },

modules/auth_saml/app/forms/saml/providers/metadata_url_form.rb

@@ -40,6 +40,14 @@ class MetadataUrlForm < BaseForm
           caption: I18n.t("saml.instructions.metadata_url"),
           input_width: :xlarge
         )
+        f.text_field(
+          name: :idp_entity_id,
+          label: I18n.t("activerecord.attributes.saml/provider.idp_entity_id"),
+          required: false,
+          disabled: provider.seeded_from_env?,
+          caption: I18n.t("saml.instructions.idp_entity_id"),
+          input_width: :xlarge
+        )
       end
     end
   end

modules/auth_saml/app/forms/saml/providers/metadata_xml_form.rb

@@ -42,6 +42,14 @@ class MetadataXmlForm < BaseForm
           rows: 10,
           input_width: :medium
         )
+        f.text_field(
+          name: :idp_entity_id,
+          label: I18n.t("activerecord.attributes.saml/provider.idp_entity_id"),
+          required: false,
+          disabled: provider.seeded_from_env?,
+          caption: I18n.t("saml.instructions.idp_entity_id"),
+          input_width: :xlarge
+        )
       end
     end
   end

modules/auth_saml/app/models/saml/provider.rb

@@ -11,6 +11,7 @@ class Provider < AuthProvider
     store_attribute :options, :metadata_xml, :string
     store_attribute :options, :last_metadata_update, :datetime
 
+    store_attribute :options, :idp_entity_id, :string
     store_attribute :options, :idp_sso_service_url, :string
     store_attribute :options, :idp_slo_service_url, :string
 

modules/auth_saml/app/services/saml/metadata_document.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+module Saml
+  # Prepares SAML metadata XML for parsing by ruby-saml.
+  #
+  # Federation aggregates MAY contain thousands of individual entities.
+  # Using ruby-saml directly would load the full document into REXML, which is extremely slow.
+  # This class streams the XML and tries to extract the matching single EntityDescriptor when we can.
+  class MetadataDocument
+    class MetadataTooLargeError < StandardError; end
+
+    class FederationMetadataError < StandardError; end
+
+    MAX_SIZE = 150.megabytes
+
+    def self.prepare(source, entity_id: nil)
+      new(source, entity_id:).prepare
+    end
+
+    def initialize(source, entity_id: nil)
+      @source = source
+      @entity_id = entity_id.presence
+    end
+
+    def prepare
+      if aggregate?
+        read_entity_fragment!
+      else
+        read_all
+      end
+    end
+
+    def read_entity_fragment!
+      fragment = extract_entity_fragment
+      if fragment.nil?
+        message =
+          if @entity_id
+            "Entity '#{@entity_id}' not found in federation aggregate"
+          else
+            "No identity provider found in federation aggregate"
+          end
+        raise FederationMetadataError, message
+      end
+
+      fragment
+    end
+
+    # Decide whether the document is a federation aggregate by inspecting its root element.
+    # Using +Nokogiri::XML::Reader+, we only advance to the root element and stop based on it.
+    def aggregate?
+      with_reader_io do |io|
+        Nokogiri::XML::Reader(io).each do |node|
+          next unless node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT
+
+          return node.local_name == "EntitiesDescriptor"
+        end
+      end
+
+      false
+    end
+
+    private
+
+    def extract_entity_fragment
+      if @entity_id
+        find_entity_by_id
+      else
+        find_first_idp_entity
+      end
+    end
+
+    # We try to prevent calling outer_xml on all entities when looking.
+    # Instead, we can only look at entityID attribute until the target is found,
+    # and then call outer_xml only on that fragment.
+    def find_entity_by_id
+      with_reader_io do |io|
+        Nokogiri::XML::Reader(io).each do |node|
+          next unless entity_descriptor_element?(node)
+          next unless node.attribute("entityID") == @entity_id
+
+          return node.outer_xml
+        end
+      end
+
+      nil
+    end
+
+    def find_first_idp_entity
+      with_reader_io do |io|
+        Nokogiri::XML::Reader(io).each do |node|
+          next unless entity_descriptor_element?(node)
+
+          fragment = node.outer_xml
+          return fragment if idp_descriptor_fragment?(fragment)
+        end
+      end
+
+      nil
+    end
+
+    def idp_descriptor_fragment?(fragment)
+      Nokogiri::XML.fragment(fragment).at_xpath(".//*[local-name()='IDPSSODescriptor']").present?
+    end
+
+    def entity_descriptor_element?(node)
+      node.node_type == Nokogiri::XML::Reader::TYPE_ELEMENT && node.local_name == "EntityDescriptor"
+    end
+
+    def read_all
+      return @source if @source.is_a?(String)
+
+      with_reader_io(&:read)
+    end
+
+    def with_reader_io(&)
+      if @source.is_a?(String)
+        StringIO.open(@source, &)
+      else
+        @source.rewind
+        yield @source
+      end
+    end
+  end
+end

modules/auth_saml/app/services/saml/metadata_fetcher.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+#-- copyright
+# OpenProject is an open source project management software.
+# Copyright (C) the OpenProject GmbH
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License version 3.
+#
+# OpenProject is a fork of ChiliProject, which is a fork of Redmine. The copyright follows:
+# Copyright (C) 2006-2013 Jean-Philippe Lang
+# Copyright (C) 2010-2013 the ChiliProject Team
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+#
+# See COPYRIGHT and LICENSE files for more details.
+#++
+
+module Saml
+  class MetadataFetcher
+    include ActionView::Helpers::NumberHelper
+
+    def self.fetch(url, &)
+      new(url).fetch(&)
+    end
+
+    def initialize(url)
+      @url = url
+    end
+
+    def fetch
+      Tempfile.create("saml-metadata") do |file|
+        file.binmode
+
+        OpenProject::SsrfProtection.get(@url) do |response|
+          unless response.is_a?(Net::HTTPSuccess)
+            raise OneLogin::RubySaml::HttpError,
+                  "Failed to fetch idp metadata: #{response.code}: #{response.message}"
+          end
+
+          bytes_written = 0
+          response.read_body do |chunk|
+            file.write(chunk)
+            bytes_written += chunk.bytesize
+            if bytes_written > MetadataDocument::MAX_SIZE
+              raise MetadataDocument::MetadataTooLargeError,
+                    "Metadata exceeds max size of #{number_to_human_size(MetadataDocument::MAX_SIZE, precision: 2)}"
+            end
+          end
+        end
+
+        file.rewind
+        yield file
+      end
+    end
+  end
+end

modules/auth_saml/app/services/saml/update_metadata_service.rb

@@ -39,8 +39,14 @@ def initialize(user:, provider:)
       @provider = provider
     end
 
-    def call
+    def call # rubocop:disable Metrics/AbcSize
       apply_metadata(merge_certificates(fetch_metadata))
+    rescue MetadataDocument::FederationMetadataError => e
+      OpenProject.logger.error(e)
+      ServiceResult.failure(result: provider, message: I18n.t("saml.metadata_parser.federation_metadata"))
+    rescue MetadataDocument::MetadataTooLargeError => e
+      OpenProject.logger.error(e)
+      ServiceResult.failure(result: provider, message: I18n.t("saml.metadata_parser.metadata_too_large"))
     rescue StandardError => e
       OpenProject.logger.error(e)
       ServiceResult.failure(result: provider,
@@ -69,12 +75,16 @@ def fetch_metadata
     end
 
     def parse_xml
-      parser_instance.parse_to_hash(provider.metadata_xml)
+      xml = MetadataDocument.prepare(provider.metadata_xml, entity_id: provider.idp_entity_id)
+      parser_instance.parse_to_hash(xml)
     end
 
     def parse_url
       validate_metadata_url_host!
-      parser_instance.parse_remote_to_hash(provider.metadata_url)
+      MetadataFetcher.fetch(provider.metadata_url) do |file|
+        xml = MetadataDocument.prepare(file, entity_id: provider.idp_entity_id)
+        parser_instance.parse_to_hash(xml)
+      end
     end
 
     def validate_metadata_url_host!

modules/auth_saml/config/locales/en.yml

@@ -12,6 +12,7 @@
         sp_entity_id: Service entity ID
         metadata_url: Identity provider metadata URL
         name_identifier_format: Name identifier format
+        idp_entity_id: Identity provider entity ID
         idp_sso_service_url: Identity provider login endpoint
         idp_slo_service_url: Identity provider logout endpoint
         idp_cert: Public certificate of identity provider
@@ -43,6 +44,10 @@
       success: "Successfully updated the configuration using the identity provider metadata."
       invalid_url: "Provided metadata URL is invalid. Provide a HTTP(s) URL."
       error: "Failed to retrieve the identity provider metadata: %{error}"
+      federation_metadata: >
+        The metadata URL points to a federation aggregate (many identity providers).
+        Use your institution's direct metadata URL, or enter your institution's IdP entity ID in the metadata form and try again.
+      metadata_too_large: "The metadata file exceeds the maximum allowed size."
     providers:
       label_empty_title: "No SAML providers configured yet."
       label_empty_description: "Add a provider to see them here."
@@ -107,6 +112,9 @@
         Your identity provider does not have a metadata endpoint or XML download option. You can configure it manually.
       metadata_url: >
         Your identity provider provides a metadata URL.
+      idp_entity_id: >
+        Optional: Useful when the metadata URL points to a federation aggregate with a lot of entries.
+        Enter the entity ID of your institution's identity provider. Leave blank for single-entity metadata URLs.
       metadata_xml: >
         Your identity provider provides a metadata XML download.
       limit_self_registration: >