Capture unknown elements into one extension tree

atom/parser.go

@@ -109,7 +109,9 @@ func (ap *Parser) parseRoot(p *xpp.Parser) (*Feed, error) {
 				atom.Entries = append(atom.Entries, entry)
 			}
 		default:
-			err = p.Skip()
+			// Not part of the spec: capture it into the extension map
+			// under the _custom pseudo namespace instead of dropping it.
+			extensions, _, err = shared.ParseCustom(extensions, p)
 		}
 		return err
 	})
@@ -215,7 +217,9 @@ func (ap *Parser) parseEntry(p *xpp.Parser) (*Entry, error) {
 		case "content":
 			entry.Content, err = ap.parseContent(p)
 		default:
-			err = p.Skip()
+			// Not part of the spec: capture it into the extension map
+			// under the _custom pseudo namespace instead of dropping it.
+			extensions, _, err = shared.ParseCustom(extensions, p)
 		}
 		return err
 	})
@@ -310,7 +314,9 @@ func (ap *Parser) parseSource(p *xpp.Parser) (*Source, error) {
 				categories = append(categories, cat)
 			}
 		default:
-			err = p.Skip()
+			// Not part of the spec: capture it into the extension map
+			// under the _custom pseudo namespace instead of dropping it.
+			extensions, _, err = shared.ParseCustom(extensions, p)
 		}
 		return err
 	})

extension_helpers.go

@@ -0,0 +1,71 @@
+package gofeed
+
+import (
+	ext "github.com/mmcdole/gofeed/extensions"
+)
+
+// CustomNamespace is the pseudo namespace under which non-namespaced unknown
+// elements are filed in Extensions, preserving nesting, attributes and
+// repetition.
+const CustomNamespace = "_custom"
+
+// GetExtension returns the extension elements for the given namespace prefix
+// and element name, or nil when absent. Non-namespaced custom elements live
+// under CustomNamespace.
+func (f *Feed) GetExtension(namespace, element string) []ext.Extension {
+	return extensionsFor(f.Extensions, namespace, element)
+}
+
+// GetExtension returns the extension elements for the given namespace prefix
+// and element name, or nil when absent. Non-namespaced custom elements live
+// under CustomNamespace.
+func (i *Item) GetExtension(namespace, element string) []ext.Extension {
+	return extensionsFor(i.Extensions, namespace, element)
+}
+
+// GetExtensionValue returns the text of the first matching extension
+// element, or "" when absent.
+func (f *Feed) GetExtensionValue(namespace, element string) string {
+	return firstExtensionValue(f.Extensions, namespace, element)
+}
+
+// GetExtensionValue returns the text of the first matching extension
+// element, or "" when absent.
+func (i *Item) GetExtensionValue(namespace, element string) string {
+	return firstExtensionValue(i.Extensions, namespace, element)
+}
+
+// GetCustomValue returns the text of the first non-namespaced custom element
+// with the given name, or "" when absent. Unlike the flat Custom map, the
+// backing extension tree also holds nested elements, attributes and repeated
+// values; use GetExtension(CustomNamespace, element) for those.
+func (f *Feed) GetCustomValue(element string) string {
+	return firstExtensionValue(f.Extensions, CustomNamespace, element)
+}
+
+// GetCustomValue returns the text of the first non-namespaced custom element
+// with the given name, or "" when absent. Unlike the flat Custom map, the
+// backing extension tree also holds nested elements, attributes and repeated
+// values; use GetExtension(CustomNamespace, element) for those.
+func (i *Item) GetCustomValue(element string) string {
+	return firstExtensionValue(i.Extensions, CustomNamespace, element)
+}
+
+func extensionsFor(exts ext.Extensions, namespace, element string) []ext.Extension {
+	if exts == nil {
+		return nil
+	}
+	nsMap, ok := exts[namespace]
+	if !ok {
+		return nil
+	}
+	return nsMap[element]
+}
+
+func firstExtensionValue(exts ext.Extensions, namespace, element string) string {
+	matches := extensionsFor(exts, namespace, element)
+	if len(matches) == 0 {
+		return ""
+	}
+	return matches[0].Value
+}

extension_helpers_test.go

@@ -0,0 +1,56 @@
+package gofeed_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mmcdole/gofeed"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExtensionHelpers(t *testing.T) {
+	feed := `<rss version="2.0" xmlns:dc="http://purl.org/dc/elements/1.1/">
+		<channel>
+			<customFeedId>feed-1</customFeedId>
+			<item>
+				<dc:creator>Jane</dc:creator>
+				<event><venue city="Austin">Hall</venue></event>
+				<simple>plain</simple>
+				<simple>again</simple>
+			</item>
+		</channel>
+	</rss>`
+
+	f, err := gofeed.NewParser().Parse(strings.NewReader(feed))
+	assert.NoError(t, err)
+	item := f.Items[0]
+
+	// Namespaced extensions through the same accessors.
+	assert.Equal(t, "Jane", item.GetExtensionValue("dc", "creator"))
+	assert.Len(t, item.GetExtension("dc", "creator"), 1)
+
+	// Feed level custom element, previously dropped entirely.
+	assert.Equal(t, "feed-1", f.GetCustomValue("customFeedId"))
+
+	// Item level custom values, including repetition the flat map loses.
+	assert.Equal(t, "plain", item.GetCustomValue("simple"))
+	assert.Len(t, item.GetExtension(gofeed.CustomNamespace, "simple"), 2)
+
+	// Nested custom elements keep children and attributes in the tree.
+	events := item.GetExtension(gofeed.CustomNamespace, "event")
+	if assert.Len(t, events, 1) {
+		venue := events[0].Children["venue"][0]
+		assert.Equal(t, "Hall", venue.Value)
+		assert.Equal(t, "Austin", venue.Attrs["city"])
+	}
+
+	// The flat Custom map still works for childless elements (last wins),
+	// and nested elements no longer corrupt it.
+	assert.Equal(t, "again", item.Custom["simple"])
+	_, hasEvent := item.Custom["event"]
+	assert.False(t, hasEvent)
+
+	// Absent lookups are empty, not panics.
+	assert.Nil(t, f.GetExtension("nope", "x"))
+	assert.Equal(t, "", item.GetCustomValue("absent"))
+}

feed.go

@@ -32,10 +32,13 @@ type Feed struct {
 	DublinCoreExt   *ext.DublinCoreExtension `json:"dcExt,omitempty"`
 	ITunesExt       *ext.ITunesFeedExtension `json:"itunesExt,omitempty"`
 	Extensions      ext.Extensions           `json:"extensions,omitempty"`
-	Custom          map[string]string        `json:"custom,omitempty"`
-	Items           []*Item                  `json:"items"`
-	FeedType        string                   `json:"feedType"`
-	FeedVersion     string                   `json:"feedVersion"`
+	// Custom is a flat view of non-namespaced unknown elements. Deprecated:
+	// use Extensions[CustomNamespace] (or GetCustomValue), which keeps
+	// nesting, attributes and repeated elements.
+	Custom      map[string]string `json:"custom,omitempty"`
+	Items       []*Item           `json:"items"`
+	FeedType    string            `json:"feedType"`
+	FeedVersion string            `json:"feedVersion"`
 
 	// originalFeed holds the source *rss.Feed, *atom.Feed, or *json.Feed when
 	// the parser was configured with KeepOriginalFeed. It is unexported (and so
@@ -81,7 +84,11 @@ type Item struct {
 	DublinCoreExt   *ext.DublinCoreExtension `json:"dcExt,omitempty"`
 	ITunesExt       *ext.ITunesItemExtension `json:"itunesExt,omitempty"`
 	Extensions      ext.Extensions           `json:"extensions,omitempty"`
-	Custom          map[string]string        `json:"custom,omitempty"`
+	// Custom is a flat view of childless non-namespaced unknown elements,
+	// last value wins. Deprecated: use Extensions[CustomNamespace] (or
+	// GetCustomValue), which keeps nesting, attributes and repeated
+	// elements.
+	Custom map[string]string `json:"custom,omitempty"`
 }
 
 // Person is an individual specified in a feed

internal/shared/extparser.go

@@ -175,3 +175,25 @@ var canonicalNamespaces = map[string]string{
 	"http://www.w3.org/XML/1998/namespace":                           "xml",
 	"http://podlove.org/simple-chapters":                             "psc",
 }
+
+// CustomPrefix is the pseudo namespace prefix that files non-namespaced
+// unknown elements in the extension map.
+const CustomPrefix = "_custom"
+
+// ParseCustom parses the current element as an arbitrary non-namespaced
+// element and files it in the extension map under CustomPrefix, preserving
+// nesting, attributes and repetition the same way namespaced extensions are
+// kept. The parsed element is also returned so callers can maintain
+// compatibility shims from it.
+func ParseCustom(fe ext.Extensions, p *xpp.Parser) (ext.Extensions, ext.Extension, error) {
+	result, err := parseExtensionElement(p)
+	if err != nil {
+		return nil, ext.Extension{}, err
+	}
+
+	if _, ok := fe[CustomPrefix]; !ok {
+		fe[CustomPrefix] = map[string][]ext.Extension{}
+	}
+	fe[CustomPrefix][result.Name] = append(fe[CustomPrefix][result.Name], result)
+	return fe, result, nil
+}

internal/shared/extparser_test.go

@@ -156,3 +156,67 @@ func TestNewXMLParserLeniencyAndCharset(t *testing.T) {
 		t.Errorf("text = %q, want café", text)
 	}
 }
+
+func TestParseCustom(t *testing.T) {
+	doc := `<rss version="2.0">
+		<channel>
+			<item>
+				<event><venue city="Austin">Hall</venue></event>
+				<simple>plain</simple>
+				<simple>again</simple>
+			</item>
+		</channel>
+	</rss>`
+
+	p := NewXMLParser(strings.NewReader(doc))
+	extensions := ext.Extensions{}
+	var parsed []ext.Extension
+	for {
+		tok, err := p.NextToken()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if tok == xpp.EndDocument {
+			break
+		}
+		if tok == xpp.StartTag {
+			switch p.Name() {
+			case "event", "simple":
+				var e ext.Extension
+				var err error
+				extensions, e, err = ParseCustom(extensions, p)
+				if err != nil {
+					t.Fatal(err)
+				}
+				parsed = append(parsed, e)
+			}
+		}
+	}
+
+	if len(parsed) != 3 {
+		t.Fatalf("parsed %d elements, want 3", len(parsed))
+	}
+
+	// Nesting and attributes survive in the tree.
+	events := extensions[CustomPrefix]["event"]
+	if len(events) != 1 {
+		t.Fatalf("event entries = %d, want 1", len(events))
+	}
+	venue := events[0].Children["venue"][0]
+	if venue.Value != "Hall" || venue.Attrs["city"] != "Austin" {
+		t.Fatalf("venue = %+v", venue)
+	}
+
+	// Repetition appends rather than overwrites.
+	if n := len(extensions[CustomPrefix]["simple"]); n != 2 {
+		t.Fatalf("simple entries = %d, want 2", n)
+	}
+
+	// The returned element mirrors what was filed.
+	if parsed[0].Name != "event" || len(parsed[0].Children) != 1 {
+		t.Fatalf("returned element = %+v", parsed[0])
+	}
+	if parsed[1].Value != "plain" || len(parsed[1].Children) != 0 {
+		t.Fatalf("returned simple = %+v", parsed[1])
+	}
+}

rss/parser.go

@@ -178,10 +178,14 @@ func (rp *Parser) parseChannel(p *xpp.Parser) (rss *Feed, err error) {
 			rss.Image, err = rp.parseImage(p)
 		case "textinput":
 			rss.TextInput, err = rp.parseTextInput(p)
-		default:
-			// Skip element as it isn't an extension and not part of
-			// the spec.
+		case "items":
+			// The RSS 1.0 <items> rdf:Seq is a structural list of item
+			// references, not content; skip it rather than capture it.
 			err = p.Skip()
+		default:
+			// Not part of the spec: capture it into the extension map
+			// under the _custom pseudo namespace instead of dropping it.
+			extensions, _, err = shared.ParseCustom(extensions, p)
 		}
 		return err
 	})
@@ -227,6 +231,26 @@ func (rp *Parser) parseItem(p *xpp.Parser) (item *Item, err error) {
 	enclosures := []*Enclosure{}
 	links := []string{}
 
+	// captureCustom files an unrecognized child in the extension map under
+	// the _custom pseudo namespace, preserving nesting and attributes, and
+	// keeps the flat Custom map populated for elements without children so
+	// existing Custom reads are unchanged.
+	captureCustom := func(p *xpp.Parser) error {
+		var e ext.Extension
+		var err error
+		extensions, e, err = shared.ParseCustom(extensions, p)
+		if err != nil {
+			return err
+		}
+		if len(e.Children) == 0 {
+			if item.Custom == nil {
+				item.Custom = make(map[string]string)
+			}
+			item.Custom[e.Name] = shared.DecodeEntities(e.Value)
+		}
+		return nil
+	}
+
 	err = shared.ForEachChild(p, func(name string) error {
 		if shared.IsExtension(p) {
 			extensions, err = shared.ParseExtension(extensions, p)
@@ -242,7 +266,7 @@ func (rp *Parser) parseItem(p *xpp.Parser) (item *Item, err error) {
 			if shared.PrefixForNamespace(p.Space(), p) == "content" {
 				item.Content, err = shared.ParseText(p)
 			} else {
-				err = rp.parseItemCustom(p, item)
+				err = captureCustom(p)
 			}
 		case "link":
 			if item.Link, err = rp.parseLink(p); err == nil {
@@ -275,7 +299,7 @@ func (rp *Parser) parseItem(p *xpp.Parser) (item *Item, err error) {
 				categories = append(categories, cat)
 			}
 		default:
-			err = rp.parseItemCustom(p, item)
+			err = captureCustom(p)
 		}
 		return err
 	})
@@ -314,21 +338,6 @@ func (rp *Parser) parseItem(p *xpp.Parser) (item *Item, err error) {
 	return item, nil
 }
 
-// parseItemCustom stores an unrecognized item child in the Custom map,
-// keyed by its original-case name. Duplicate names keep the last value.
-func (rp *Parser) parseItemCustom(p *xpp.Parser, item *Item) error {
-	key := p.Name()
-	result, err := shared.ParseText(p)
-	if err != nil {
-		return err
-	}
-	if item.Custom == nil {
-		item.Custom = make(map[string]string)
-	}
-	item.Custom[key] = result
-	return nil
-}
-
 func (rp *Parser) parseLink(p *xpp.Parser) (url string, err error) {
 	base := p.BaseURL()
 	href := p.Attribute("href")