Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -152,3 +152,4 @@ db/
start.sh
start-full.sh
.claude/
.dev-data/
2 changes: 1 addition & 1 deletion internal/controller/feed_viewer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ func TestPreviewFeedViewerSupportsTopicURI(t *testing.T) {
ID: topicID,
Title: "Preview Topic",
Description: "Topic preview",
InputURIs: []string{"feedcraft://recipe/" + recipeID},
Inputs: []dao.TopicInput{{URI: "feedcraft://recipe/" + recipeID}},
})

recorder := performFeedViewerPreviewRequest(t, http.MethodGet, "feedcraft://topic/"+topicID)
Expand Down
23 changes: 14 additions & 9 deletions internal/controller/topic_feed.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ func CreateTopicFeed(c *gin.Context) {
}
db := util.GetDatabase()

topicData.NormalizeInputs()
if err := dao.CreateTopicFeed(db, &topicData); err != nil {
c.JSON(http.StatusInternalServerError, util.APIResponse[any]{Msg: err.Error()})
return
Expand Down Expand Up @@ -129,6 +130,7 @@ func UpdateTopicFeed(c *gin.Context) {
return
}

topicData.NormalizeInputs()
if err := dao.UpdateTopicFeed(db, &topicData); err != nil {
c.JSON(http.StatusInternalServerError, util.APIResponse[any]{Msg: err.Error()})
return
Expand Down Expand Up @@ -211,9 +213,9 @@ func GetTopicFeedDetail(c *gin.Context) {
return
}

subFeedHealth := make([]feedruntime.SubFeedHealth, 0, len(topicData.InputURIs))
for _, uri := range topicData.InputURIs {
subFeedHealth = append(subFeedHealth, feedruntime.GetSubFeedHealth(uri))
subFeedHealth := make([]feedruntime.SubFeedHealth, 0, len(topicData.Inputs))
for _, input := range topicData.Inputs {
subFeedHealth = append(subFeedHealth, feedruntime.GetSubFeedHealth(input.URI))
}

detail := TopicDetailResponse{
Expand Down Expand Up @@ -273,6 +275,9 @@ func validateTopicConfig(ctx context.Context, db *gorm.DB, topicData *dao.TopicF
Errors: []TopicValidationIssue{},
Warnings: []TopicValidationIssue{},
}
if topicData != nil {
topicData.NormalizeInputs()
}
if topicData == nil {
result.Valid = false
result.Errors = append(result.Errors, TopicValidationIssue{
Expand All @@ -291,19 +296,19 @@ func validateTopicConfig(ctx context.Context, db *gorm.DB, topicData *dao.TopicF
return result, nil
}

if len(topicData.InputURIs) == 0 {
if len(topicData.EnabledInputURIs()) == 0 {
result.Valid = false
result.Errors = append(result.Errors, TopicValidationIssue{
Field: "input_uris",
Message: "At least one input source is required",
Field: "inputs",
Message: "At least one enabled input source is required",
})
}

for idx, uri := range topicData.InputURIs {
if strings.TrimSpace(uri) == "" {
for idx, input := range topicData.Inputs {
if strings.TrimSpace(input.URI) == "" {
result.Valid = false
result.Errors = append(result.Errors, TopicValidationIssue{
Field: fmt.Sprintf("input_uris[%d]", idx),
Field: fmt.Sprintf("inputs[%d].uri", idx),
Message: "Input URI cannot be empty",
})
}
Expand Down
28 changes: 14 additions & 14 deletions internal/controller/topic_feed_public_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ func TestPublicTopicFeed(t *testing.T) {
ID: topicID,
Title: "My Topic",
Description: "Topic description",
InputURIs: []string{"feedcraft://recipe/" + recipeID},
Inputs: []dao.TopicInput{{URI: "feedcraft://recipe/" + recipeID}},
})

req, err := http.NewRequest(http.MethodGet, "/topic/"+topicID, nil)
Expand Down Expand Up @@ -105,9 +105,9 @@ func TestPublicTopicFeed(t *testing.T) {
t.Run("returns 500 when topic build fails", func(t *testing.T) {
topicID := uniqueTestID("topic-invalid")
createTopicTestTopic(t, db, &dao.TopicFeed{
ID: topicID,
Title: "Broken Topic",
InputURIs: []string{"feedcraft://broken/abc"},
ID: topicID,
Title: "Broken Topic",
Inputs: []dao.TopicInput{{URI: "feedcraft://broken/abc"}},
})

req, err := http.NewRequest(http.MethodGet, "/topic/"+topicID, nil)
Expand All @@ -125,9 +125,9 @@ func TestPublicTopicFeed(t *testing.T) {
topicID := uniqueTestID("topic-partial")
createTopicTestRecipe(t, db, recipeID)
createTopicTestTopic(t, db, &dao.TopicFeed{
ID: topicID,
Title: "Partial Topic",
InputURIs: []string{"feedcraft://recipe/" + recipeID, "http://127.0.0.1:1/unreachable.xml"},
ID: topicID,
Title: "Partial Topic",
Inputs: []dao.TopicInput{{URI: "feedcraft://recipe/" + recipeID}, {URI: "http://127.0.0.1:1/unreachable.xml"}},
})

req, err := http.NewRequest(http.MethodGet, "/topic/"+topicID, nil)
Expand All @@ -144,9 +144,9 @@ func TestPublicTopicFeed(t *testing.T) {
t.Run("returns 500 when all upstreams fail", func(t *testing.T) {
topicID := uniqueTestID("topic-all-failed")
createTopicTestTopic(t, db, &dao.TopicFeed{
ID: topicID,
Title: "Failed Topic",
InputURIs: []string{"http://127.0.0.1:1/a.xml", "http://127.0.0.1:1/b.xml"},
ID: topicID,
Title: "Failed Topic",
Inputs: []dao.TopicInput{{URI: "http://127.0.0.1:1/a.xml"}, {URI: "http://127.0.0.1:1/b.xml"}},
})

req, err := http.NewRequest(http.MethodGet, "/topic/"+topicID, nil)
Expand Down Expand Up @@ -182,7 +182,7 @@ func TestTopicFeedAdminEndpoints(t *testing.T) {
body := `{
"id":"` + uniqueTestID("topic-validate-ok") + `",
"title":"Tech Topic",
"input_uris":["feedcraft://recipe/` + recipeID + `"],
"inputs":[{"uri":"feedcraft://recipe/` + recipeID + `"}],
"aggregator_config":[{"type":"limit","option":{"max":"10"}}]
}`

Expand All @@ -206,7 +206,7 @@ func TestTopicFeedAdminEndpoints(t *testing.T) {
body := `{
"id":"` + uniqueTestID("topic-invalid-step") + `",
"title":"Broken Step Topic",
"input_uris":["feedcraft://recipe/` + recipeID + `"],
"inputs":[{"uri":"feedcraft://recipe/` + recipeID + `"}],
"aggregator_config":[{"type":"limit","option":{"max":"0"}}]
}`

Expand All @@ -230,7 +230,7 @@ func TestTopicFeedAdminEndpoints(t *testing.T) {
body := `{
"id":"` + topicID + `",
"title":"Cycle Topic",
"input_uris":["feedcraft://topic/` + topicID + `"],
"inputs":[{"uri":"feedcraft://topic/` + topicID + `"}],
"aggregator_config":[{"type":"limit","option":{"max":"10"}}]
}`

Expand All @@ -255,7 +255,7 @@ func TestTopicFeedAdminEndpoints(t *testing.T) {
ID: topicID,
Title: "Detail Topic",
Description: "detail description",
InputURIs: []string{"https://example.com/feed.xml"},
Inputs: []dao.TopicInput{{URI: "https://example.com/feed.xml"}},
AggregatorConfig: []dao.AggregatorStep{
{Type: "limit", Option: map[string]string{"max": "20"}},
},
Expand Down
56 changes: 56 additions & 0 deletions internal/dao/migrate.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@

// Perform data migration from custom_recipes to custom_recipes_v2
migrateRecipesToV2(db)
migrateTopicFeedInputs(db)

logrus.Info("migrate database done.")

Expand Down Expand Up @@ -98,6 +99,61 @@
logrus.Info("recipe migration to v2 completed.")
}

func migrateTopicFeedInputs(db *gorm.DB) {

Check warning on line 102 in internal/dao/migrate.go

View check run for this annotation

Codacy Production / Codacy Static Code Analysis

internal/dao/migrate.go#L102

Method migrateTopicFeedInputs has a cyclomatic complexity of 16 (limit is 8)
if !db.Migrator().HasTable(&TopicFeed{}) {
return
}
if !db.Migrator().HasColumn("topic_feeds", "input_uris") {
return
}

type legacyTopicInputRow struct {
ID string
Inputs string
InputURIs string
}

var rows []legacyTopicInputRow
if err := db.Table("topic_feeds").Select("id, inputs, input_uris").Scan(&rows).Error; err != nil {
logrus.Errorf("failed to scan topic feed input_uris migration rows: %v", err)
return
}

for _, row := range rows {
if row.InputURIs == "" || row.InputURIs == "null" || row.InputURIs == "[]" {
continue
}
if row.Inputs != "" && row.Inputs != "null" && row.Inputs != "[]" {
continue
}

var uris []string
if err := json.Unmarshal([]byte(row.InputURIs), &uris); err != nil {
logrus.Errorf("failed to unmarshal legacy input_uris for topic %s: %v", row.ID, err)
continue
}

inputs := make([]TopicInput, 0, len(uris))
for _, uri := range uris {
inputs = append(inputs, TopicInput{URI: uri})
}
inputsJSON, err := json.Marshal(inputs)
if err != nil {
logrus.Errorf("failed to marshal migrated inputs for topic %s: %v", row.ID, err)
continue
}
if err := db.Table("topic_feeds").Where("id = ?", row.ID).Update("inputs", string(inputsJSON)).Error; err != nil {
logrus.Errorf("failed to migrate topic inputs for topic %s: %v", row.ID, err)
}
}

if err := db.Exec("ALTER TABLE topic_feeds DROP COLUMN input_uris").Error; err != nil {
logrus.Errorf("failed to drop legacy topic input_uris column: %v", err)
return
}
logrus.Info("topic feed input_uris migration completed.")
}

var defaultAdminUsername = "admin"
var defaultPassword = "adminadmin" // default defaultPassword string

Expand Down
56 changes: 49 additions & 7 deletions internal/dao/topic.go
Original file line number Diff line number Diff line change
@@ -1,28 +1,70 @@
package dao

import (
"strings"

"gorm.io/gorm"
)

// TopicInput is a single upstream source with optional admin metadata.
type TopicInput struct {
URI string `json:"uri"`
Description string `json:"description,omitempty"`
// Disabled excludes this input from topic aggregation when true.
Disabled bool `json:"disabled,omitempty"`
}

// TopicFeed represents the persistence model for a multi-source aggregation node.
type TopicFeed struct {
BaseModelWithoutPK
ID string `gorm:"primaryKey" json:"id" binding:"required"`
Title string `json:"title,omitempty"`
Description string `json:"description,omitempty"`

// List of URIs representing inputs.
// Uses a custom protocol for internal resources to make routing elegant and standard.
// Examples:
// - "feedcraft://recipe/my-tech-recipe" (Internal RecipeFeed)
// - "feedcraft://topic/sub-topic-id" (Nested internal TopicFeed)
// - "https://external.com/rss.xml" (External raw feed)
InputURIs []string `json:"input_uris" binding:"required" gorm:"serializer:json"`
// Inputs carries URI plus optional description for admin display.
Inputs []TopicInput `json:"inputs,omitempty" gorm:"serializer:json"`

// Configuration for the aggregator pipeline
AggregatorConfig []AggregatorStep `json:"aggregator_config" gorm:"serializer:json"`
}

// NormalizeInputs trims input metadata and removes blank URIs.
func (t *TopicFeed) NormalizeInputs() {
if t == nil {
return
}

normalized := make([]TopicInput, 0, len(t.Inputs))
for _, item := range t.Inputs {
uri := strings.TrimSpace(item.URI)
if uri == "" {
continue
}
normalized = append(normalized, TopicInput{
URI: uri,
Description: strings.TrimSpace(item.Description),
Disabled: item.Disabled,
})
}
t.Inputs = normalized
}

// EnabledInputURIs returns input URIs that participate in topic aggregation.
func (t *TopicFeed) EnabledInputURIs() []string {
if t == nil {
return nil
}
uris := make([]string, 0, len(t.Inputs))
for _, item := range t.Inputs {
uri := strings.TrimSpace(item.URI)
if uri == "" || item.Disabled {
continue
}
uris = append(uris, uri)
}
return uris
}

// AggregatorStep defines a single processing step in an Aggregator pipeline.
type AggregatorStep struct {
Type string `json:"type" binding:"required"` // e.g., "deduplicate", "sort", "limit"
Expand Down
Loading
Loading