feat(app): add internal components batch 5

This commit is contained in:
吴璨
2026-09-30 10:20:38 +08:00
parent 4ec1f85bbc
commit 5bd126e026
10 changed files with 1956 additions and 0 deletions
+170
View File
@@ -0,0 +1,170 @@
package app
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"time"
)
type libraryCache struct {
Dramas []Drama `json:"dramas"`
LoadedAt time.Time `json:"loadedAt"`
LastError string `json:"lastError,omitempty"`
Sources map[string]librarySourceState `json:"sources,omitempty"`
HongguoApp *hongguoCatalogState `json:"hongguoApp,omitempty"`
}
func libraryCachePath(outputDir string) string {
return filepath.Join(outputDir, "library.json")
}
func readLibraryCache(outputDir string) (libraryCache, error) {
var cache libraryCache
path := libraryCachePath(outputDir)
body, err := os.ReadFile(path)
if errors.Is(err, os.ErrNotExist) {
path = filepath.Join(outputDir, "ui-state.json")
body, err = os.ReadFile(path)
}
if err != nil {
return cache, err
}
if err := json.Unmarshal(body, &cache); err != nil {
return cache, fmt.Errorf("本地剧库缓存损坏 %s: %w", path, err)
}
originalCount := len(cache.Dramas)
cache.Dramas = onlyHongguoDramas(cache.Dramas)
sources := make(map[string]librarySourceState)
for source, state := range cache.Sources {
if canonicalProviderSource(source) == sourceHongguo {
sources[sourceHongguo] = state
}
}
if originalCount != len(cache.Dramas) || len(sources) != len(cache.Sources) {
cache.LastError = librarySourceErrors(sources)
}
cache.Sources = sources
return cache, nil
}
func writeLibraryCache(outputDir string, cache libraryCache) error {
if err := os.MkdirAll(outputDir, 0o755); err != nil {
return err
}
file, err := os.CreateTemp(outputDir, ".library-*.tmp")
if err != nil {
return err
}
defer os.Remove(file.Name())
encoder := json.NewEncoder(file)
if err := encoder.Encode(cache); err != nil {
_ = file.Close()
return err
}
if err := file.Sync(); err != nil {
_ = file.Close()
return err
}
if err := file.Close(); err != nil {
return err
}
return os.Rename(file.Name(), libraryCachePath(outputDir))
}
func (a *UIApp) loadLibrary() {
cache, err := readLibraryCache(a.cfg.dataDirectory())
if err == nil {
a.downloader.restoreHongguoCatalog(cache.HongguoApp)
}
a.mu.Lock()
defer a.mu.Unlock()
if err != nil {
if !errors.Is(err, os.ErrNotExist) {
a.libraryError = a.redactError(err)
}
return
}
if len(cache.Dramas) > 0 && (len(a.dramas) == 0 || !a.loadedAt.After(cache.LoadedAt)) {
a.dramas = cache.Dramas
a.normalizeDramaCovers(a.dramas)
a.loadedAt = cache.LoadedAt
a.libraryError = cache.LastError
}
a.librarySources = cache.Sources
if cache.HongguoApp != nil {
a.libraryApp = a.downloader.hongguoCatalogSnapshot()
}
for source, state := range a.librarySources {
if state.Status == "loading" {
state.Status = "partial"
state.Error = "上次更新中断,当前显示已保存数据,可手动刷新"
a.librarySources[source] = state
}
}
if sourceErrors := librarySourceErrors(a.librarySources); sourceErrors != "" {
a.libraryError = sourceErrors
}
a.libraryRevision = 1
if len(a.dramas) > 0 {
if _, err := os.Stat(libraryCachePath(a.cfg.dataDirectory())); errors.Is(err, os.ErrNotExist) {
if err := writeLibraryCache(a.cfg.dataDirectory(), libraryCache{Dramas: a.dramas, LoadedAt: a.loadedAt, LastError: a.libraryError, Sources: a.librarySources, HongguoApp: a.libraryApp}); err != nil {
a.libraryError = "剧库缓存保存失败: " + a.redactError(err)
return
}
}
a.librarySaved = true
var pending []Drama
for _, drama := range a.dramas {
if drama.SortMetadata != nil && drama.SortMetadata.Pending {
pending = append(pending, drama)
}
}
a.enqueueSortMetadataLocked(pending, false)
fmt.Printf("已恢复本地剧库:%d 部,缓存:%s\n", len(a.dramas), libraryCachePath(a.cfg.dataDirectory()))
}
}
func (d *Downloader) GetAllDramas(ctx context.Context) ([]Drama, error) {
cache, err := readLibraryCache(d.cfg.dataDirectory())
if err == nil {
d.restoreHongguoCatalog(cache.HongguoApp)
}
if err == nil && len(cache.Dramas) > 0 {
fmt.Printf("使用本地剧库缓存:%d 部(%s),更新请使用 -refresh\n", len(cache.Dramas), cache.LoadedAt.Format("2006-01-02 15:04:05"))
if cache.LastError != "" {
return cache.Dramas, errors.New(cache.LastError)
}
return cache.Dramas, nil
}
if err != nil && !errors.Is(err, os.ErrNotExist) {
fmt.Printf(" 读取本地剧库缓存失败: %v\n", err)
}
return d.RefreshDramas(ctx, "")
}
func (d *Downloader) RefreshDramas(ctx context.Context, source string) ([]Drama, error) {
cached, _ := readLibraryCache(d.cfg.dataDirectory())
d.restoreHongguoCatalog(cached.HongguoApp)
ctx = withKnownHongguoDramas(ctx, cached.Dramas)
fresh, loadErr := d.fetchAllDramas(ctx, source)
merged := mergeSourceDramas(cached.Dramas, fresh, loadErr, source)
if len(fresh) == 0 && len(merged) == 0 {
return nil, loadErr
}
cache := libraryCache{Dramas: merged, LoadedAt: time.Now(), HongguoApp: d.hongguoCatalogSnapshot()}
if len(fresh) == 0 {
cache.LoadedAt = cached.LoadedAt
}
if loadErr != nil {
cache.LastError = publicError(loadErr).Error()
}
if err := writeLibraryCache(d.cfg.dataDirectory(), cache); err != nil {
loadErr = errors.Join(loadErr, fmt.Errorf("剧库缓存保存失败: %w", err))
}
fmt.Printf("本地剧库:%d 部,本次新增 %d 部\n", len(merged), len(merged)-len(cached.Dramas))
return merged, loadErr
}
+109
View File
@@ -0,0 +1,109 @@
package app
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"sync/atomic"
"testing"
"time"
)
const coverAddressFixture = "https://p3-reading-sign.fqnovelpic.com/synthetic-cover-address?fixture=1"
func TestHongguoRankingKeepsCoverAddressForLibrary(t *testing.T) {
board, _ := findRankingBoard("hongguo-hot")
rows := rankingRows(1)
rows[0]["cover"] = coverAddressFixture
page, err := parseHongguoRanking(rankingFixture(board, 1, rows), board, 1)
if err != nil || bestDramaCover(page.Items[0].Drama) != coverAddressFixture || bestDramaCover(page.Items[1].Drama) != "" {
t.Fatal("ranking discarded its cover address or invented one", err)
}
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
t.Error("retaining an address must not fetch the image", request.URL.Path)
return rankingHTTPResponse(request, 404, ""), nil
})
app := &UIApp{downloader: d, cfg: d.cfg, metadataClosed: true}
page.BoardID, page.Page, page.FetchedAt = board.ID, 1, time.Now()
app.acceptRankingDramas(page)
for _, drama := range app.dramas {
if drama.ID == page.Items[0].Drama.ID && drama.Cover != "/api/ui/image?url="+url.QueryEscape(coverAddressFixture)+"&dramaId="+url.QueryEscape(drama.ID) {
t.Fatal("ranking address bypassed the local cover route")
}
}
for _, invalid := range []any{"javascript:invalid", "https://example.invalid/cover", "http://p3-reading-sign.fqnovelpic.com/cover", map[string]any{"unexpected": "field"}} {
rows[0]["cover"] = invalid
result, err := parseHongguoRanking(rankingFixture(board, 1, rows), board, 1)
if err != nil || len(result.Items) != 2 || bestDramaCover(result.Items[0].Drama) != "" {
t.Fatal("invalid optional address broke the ranking or reached the library", err)
}
}
}
func TestHistoricalMissingCoverWaitsForManualUpdate(t *testing.T) {
for _, found := range []bool{true, false} {
t.Run(fmt.Sprintf("address=%t", found), func(t *testing.T) {
var calls atomic.Int32
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
calls.Add(1)
if request.URL.Path != "/novel/player/video_detail/v1/" || request.Context().Value(backgroundCatalogKey{}) != true {
t.Error("cover metadata used a non-background or non-text request", request.URL.Path)
return rankingHTTPResponse(request, 404, ""), nil
}
cover := ""
if found {
cover = coverAddressFixture
}
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":"7000000000000000001","series_title":"合成资料","series_cover":%q,"hot_score":999,"series_play_cnt":999}}}`, cover)), nil
})
old := Drama{ID: "hongguo:7000000000000000001", Source: sourceHongguo, Title: "历史榜单条目", OnlineDate: "2026-09-01", Heat: "20019", Views: "0",
SortMetadata: &sortMetadataState{Version: sortMetadataVersion, CheckedAt: time.Now().Add(-time.Hour)}}
if err := writeLibraryCache(d.cfg.dataDirectory(), libraryCache{Dramas: []Drama{old}}); err != nil {
t.Fatal(err)
}
app := &UIApp{downloader: d, cfg: d.cfg}
t.Cleanup(app.stopSortMetadata)
app.loadLibrary()
app.handleDramas(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/ui/dramas", nil))
app.mu.Lock()
if calls.Load() != 0 || app.metadataDone != nil || len(app.newSortMetadataDramasLocked([]Drama{old})) != 0 || sortMetadataRemaining(app.dramas)[sourceHongguo] != 1 {
app.mu.Unlock()
t.Fatal("ordinary loading migrated historical addresses or hid their pending update")
}
app.enqueueHistoricalMetadataLocked(sourceHongguo, []string{old.ID})
app.mu.Unlock()
waitSortMetadataQueue(t, app)
cache, err := readLibraryCache(d.cfg.dataDirectory())
if err != nil || len(cache.Dramas) != 1 {
t.Fatal("manual address repair was not persisted", err)
}
updated := cache.Dramas[0]
if calls.Load() != 1 || updated.SortMetadata == nil || !updated.SortMetadata.CoverChecked || needsSortMetadata(updated) || updated.Views != "0" || updated.Heat != old.Heat || updated.Title != old.Title {
t.Fatal("cover repair changed primary metadata or did not remember a completed check")
}
if found && (bestDramaCover(updated) != coverAddressFixture || !strings.HasPrefix(updated.Cover.(string), "/api/ui/image?url=")) || !found && bestDramaCover(updated) != "" {
t.Fatal("manual repair lost the address or invented a missing cover")
}
app.mu.Lock()
app.enqueueHistoricalMetadataLocked(sourceHongguo, []string{old.ID})
app.mu.Unlock()
waitSortMetadataQueue(t, app)
if calls.Load() != 1 {
t.Fatal("known missing cover triggered repeated requests")
}
})
}
}
func TestHongguoCoverRepairValidatesDramaIdentity(t *testing.T) {
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":"7000000000000000002","series_cover":%q}}}`, coverAddressFixture)), nil
})
patch, err := d.fetchDramaSortMetadata(context.Background(), Drama{ID: "hongguo:7000000000000000001", OnlineDate: "2026-09-01", Heat: "1", Views: "0"})
if err == nil || bestDramaCover(patch) != "" {
t.Fatal("cover repair accepted another drama's address")
}
}
+98
View File
@@ -0,0 +1,98 @@
package app
import (
"context"
"errors"
"fmt"
"time"
)
func (d *Downloader) fetchAllDramas(ctx context.Context, sourceFilter string) ([]Drama, error) {
if sourceFilter != "" && canonicalProviderSource(sourceFilter) != sourceHongguo {
return nil, errors.New("此版本仅支持红果站源")
}
if more, _ := ctx.Value(libraryMoreKey{}).(bool); more {
return d.fetchMoreLibrary(ctx, sourceFilter)
}
category, _ := ctx.Value(libraryCategoryKey{}).(string)
fmt.Println("正在获取红果剧库列表...")
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
defer cancel()
dramas, err := d.fetchHongguoCategoryDramas(ctx, category)
dramas = onlyHongguoDramas(dramas)
if len(dramas) == 0 && err == nil && !hongguoCatalogInitialized(d.hongguoCatalogSnapshot()) {
err = errors.New("未返回可识别的视频数据")
}
reportLibraryProgress(ctx, sourceHongguo, dramas, err, true)
fmt.Printf(" 剧库获取完成:%d 部\n", len(dramas))
if err != nil {
fmt.Printf(" 红果获取失败: %v\n", publicError(err))
return dramas, &libraryLoadError{failures: map[string]error{sourceHongguo: err}}
}
return dramas, nil
}
func (d *Downloader) GetDramaChapters(ctx context.Context, seriesID string) (string, []Chapter, error) {
_, sourceID, valid := splitProviderDramaID(seriesID)
if !valid {
return "", nil, errors.New("仅支持红果剧集 ID,可填写 hongguo:数字ID 或数字ID")
}
title, chapters, err := d.fetchHongguoChapters(ctx, sourceID)
return title, uniqueChapters(chapters), err
}
func uniqueChapters(chapters []Chapter) []Chapter {
seen := map[string]bool{}
var out []Chapter
for i, ch := range chapters {
key := ch.ID
if key == "" {
key = ch.VideoURL
}
if key == "" {
key = fmt.Sprintf("idx_%d", i)
}
if seen[key] {
continue
}
seen[key] = true
out = append(out, ch)
}
return out
}
func (d *Downloader) BuildDramaTasks(ctx context.Context, drama Drama) ([]Task, error) {
return d.buildDramaTasksInDirectory(ctx, drama, "")
}
func (d *Downloader) buildDramaTasksInDirectory(ctx context.Context, drama Drama, existingDirectory string) ([]Task, error) {
var valid bool
drama, valid = normalizeHongguoDrama(drama)
if !valid {
return nil, errors.New("此版本仅支持红果剧集")
}
title, chapters, err := d.GetDramaChapters(ctx, drama.ID)
if err != nil {
return nil, err
}
if len(chapters) == 0 {
return nil, nil
}
drama = d.hongguoCachedDrama(drama)
dramaDir, err := d.downloadDramaDirectory(drama, title, existingDirectory)
if err != nil {
return nil, err
}
var tasks []Task
for i, ch := range chapters {
ep := ch.EpisodeString(i + 1)
epNum := padEpisode(ep)
fileName := safeFilename(epNum + ".mp4")
outPath, err := safeJoin(dramaDir, fileName)
if err != nil {
return nil, err
}
tasks = append(tasks, Task{DramaID: drama.ID, DramaTitle: title, Chapter: ch, Index: i + 1, Total: len(chapters), OutPath: outPath, ReleaseStatus: dramaReleaseStatus(drama)})
}
return tasks, nil
}
+252
View File
@@ -0,0 +1,252 @@
package app
import (
"context"
"errors"
"fmt"
"net/http"
"sort"
"strings"
"sync"
"time"
)
const sortMetadataVersion = 1
const sortMetadataBatchSize = 40
const sortMetadataRetryDelay = 5 * time.Minute
const sortMetadataMissingRetryDelay = 24 * time.Hour
type sortMetadataState struct {
Version int `json:"version"`
CheckedAt time.Time `json:"checkedAt"`
Pending bool `json:"pending,omitempty"`
CoverChecked bool `json:"coverChecked,omitempty"`
RetryAt time.Time `json:"retryAt,omitempty"`
LastError string `json:"lastError,omitempty"`
}
type libraryRowsKey struct{}
type libraryPriorityKey struct{}
type libraryMetadataProgressKey struct{}
type libraryMetadataProgress struct {
Running bool `json:"running"`
Checked int `json:"checked"`
Total int `json:"total"`
Updated int `json:"updated"`
Failed int `json:"failed"`
}
func supportsSortMetadata(drama Drama) bool {
switch dramaProvider(drama) {
case sourceHongguo:
return true
}
return false
}
func needsSortMetadata(drama Drama) bool {
return supportsSortMetadata(drama) && (drama.SortMetadata == nil || drama.SortMetadata.Version != sortMetadataVersion || needsHongguoCoverAddress(drama))
}
func sortMetadataRetryAt(state *sortMetadataState, now time.Time) time.Time {
if state == nil || state.Version != 0 {
return time.Time{}
}
if !state.RetryAt.IsZero() {
return state.RetryAt
}
if state.CheckedAt.IsZero() {
return time.Time{}
}
return state.CheckedAt.Add(sortMetadataRetryDelay)
}
func sortMetadataFailureDelay(err error) time.Duration {
var status *httpStatusError
if errors.As(err, &status) && (status.status == http.StatusNotFound || status.status == http.StatusGone) {
return sortMetadataMissingRetryDelay
}
text := strings.ToLower(err.Error())
if strings.Contains(text, "无效的") || strings.Contains(text, "未返回所请求") || strings.Contains(text, "返回了其他剧集") || strings.Contains(text, "与请求剧集不符") {
return sortMetadataMissingRetryDelay
}
return sortMetadataRetryDelay
}
func failedSortMetadataState(drama Drama, err error, now time.Time) *sortMetadataState {
state := &sortMetadataState{CheckedAt: now, RetryAt: now.Add(sortMetadataFailureDelay(err)), LastError: truncate(publicError(err).Error(), 300)}
if drama.SortMetadata != nil {
state.CoverChecked = drama.SortMetadata.CoverChecked
}
return state
}
func needsHongguoCoverAddress(drama Drama) bool {
return dramaProvider(drama) == sourceHongguo && bestDramaCover(drama) == "" &&
(drama.SortMetadata == nil || !drama.SortMetadata.CoverChecked)
}
func selectSortMetadataBatch(dramas []Drama, source string, priority []string, now time.Time) []Drama {
positions := make(map[string]int, len(priority))
for index, id := range priority {
if _, found := positions[id]; !found {
positions[id] = index
}
}
var candidates []Drama
seen := map[string]bool{}
for _, drama := range dramas {
if seen[drama.ID] || !matchesSourceFilter(dramaProvider(drama), source) || !needsSortMetadata(drama) {
continue
}
seen[drama.ID] = true
if retryAt := sortMetadataRetryAt(drama.SortMetadata, now); !retryAt.IsZero() && now.Before(retryAt) {
continue
}
candidates = append(candidates, drama)
}
sort.SliceStable(candidates, func(i, j int) bool {
left, leftOK := positions[candidates[i].ID]
right, rightOK := positions[candidates[j].ID]
if leftOK != rightOK {
return leftOK
}
if leftOK && left != right {
return left < right
}
var leftTime, rightTime time.Time
if candidates[i].SortMetadata != nil {
leftTime = candidates[i].SortMetadata.CheckedAt
}
if candidates[j].SortMetadata != nil {
rightTime = candidates[j].SortMetadata.CheckedAt
}
return leftTime.Before(rightTime)
})
if len(candidates) > sortMetadataBatchSize {
candidates = candidates[:sortMetadataBatchSize]
}
return candidates
}
func sortMetadataRemaining(dramas []Drama) map[string]int {
counts := map[string]int{"": 0, "hongguo": 0}
for _, drama := range dramas {
if !needsSortMetadata(drama) {
continue
}
source := dramaProvider(drama)
counts[source]++
counts[""]++
}
return counts
}
func reportMetadataProgress(ctx context.Context, progress libraryMetadataProgress) {
if callback, ok := ctx.Value(libraryMetadataProgressKey{}).(func(libraryMetadataProgress)); ok {
callback(progress)
}
}
func (d *Downloader) backfillSortMetadata(ctx context.Context, source string) ([]Drama, map[string]error) {
rows, _ := ctx.Value(libraryRowsKey{}).([]Drama)
priority, _ := ctx.Value(libraryPriorityKey{}).([]string)
batch := selectSortMetadataBatch(rows, source, priority, time.Now())
progress := libraryMetadataProgress{Running: true, Total: len(batch)}
reportMetadataProgress(ctx, progress)
defer func() { progress.Running = false; reportMetadataProgress(ctx, progress) }()
failures := map[string]error{}
if len(batch) == 0 {
return nil, failures
}
workCtx, cancel := context.WithTimeout(ctx, 3*time.Minute)
defer cancel()
type result struct {
patch Drama
previous Drama
err error
}
jobs := make(chan Drama, len(batch))
results := make(chan result, len(batch))
for _, drama := range batch {
jobs <- drama
}
close(jobs)
workers := d.cfg.RequestConcurrency
if workers < 1 {
workers = 1
}
if workers > 3 {
workers = 3
}
var group sync.WaitGroup
for worker := 0; worker < workers; worker++ {
group.Add(1)
go func() {
defer group.Done()
for drama := range jobs {
if workCtx.Err() != nil {
return
}
itemCtx, stop := context.WithTimeout(workCtx, 40*time.Second)
patch, err := d.fetchDramaSortMetadata(itemCtx, drama)
stop()
now := time.Now()
patch.SortMetadata = &sortMetadataState{CheckedAt: now}
if err == nil {
patch.SortMetadata.Version = sortMetadataVersion
patch.SortMetadata.CoverChecked = true
} else {
patch.SortMetadata = failedSortMetadataState(drama, err, now)
}
results <- result{patch: patch, previous: drama, err: err}
}
}()
}
go func() { group.Wait(); close(results) }()
var patches []Drama
for result := range results {
patch := result.patch
provider := dramaProvider(result.previous)
if result.err != nil {
progress.Failed++
if failures[provider] == nil {
failures[provider] = fmt.Errorf("部分历史资料未补齐,可稍后重试: %w", result.err)
}
}
if patch.OnlineDate != "" && patch.OnlineDate != result.previous.OnlineDate || patch.Heat != "" && patch.Heat != result.previous.Heat || patch.Views != "" && patch.Views != result.previous.Views || bestDramaCover(patch) != "" && bestDramaCover(patch) != bestDramaCover(result.previous) {
progress.Updated++
}
patches = append(patches, patch)
progress.Checked++
reportLibraryProgress(ctx, provider, []Drama{patch}, nil, false)
reportMetadataProgress(ctx, progress)
}
if progress.Checked < len(batch) {
for _, drama := range batch {
provider := dramaProvider(drama)
if failures[provider] == nil {
failures[provider] = errors.New("本批补齐已暂停,剩余条目可再次手动补齐")
}
}
}
return patches, failures
}
func (d *Downloader) fetchMoreLibrary(ctx context.Context, source string) ([]Drama, error) {
category, _ := ctx.Value(libraryCategoryKey{}).(string)
patches, failures := d.backfillSortMetadata(ctx, source)
if matchesSourceFilter(sourceHongguo, source) {
items, err := d.fetchHongguoCategoryDramas(ctx, category)
if err != nil {
failures[sourceHongguo] = errors.Join(failures[sourceHongguo], err)
}
reportLibraryProgress(ctx, sourceHongguo, items, failures[sourceHongguo], true)
patches = mergeSourceDramas(patches, items, nil, sourceHongguo)
}
if len(failures) > 0 {
return patches, &libraryLoadError{failures: failures}
}
return patches, nil
}
+185
View File
@@ -0,0 +1,185 @@
package app
import (
"context"
"time"
)
func (a *UIApp) newSortMetadataDramasLocked(items []Drama) []Drama {
known := make(map[string]bool, len(a.dramas))
for _, drama := range a.dramas {
known[drama.ID] = true
}
var fresh []Drama
for _, drama := range items {
if drama.ID != "" && !known[drama.ID] {
known[drama.ID] = true
if needsSortMetadata(drama) {
fresh = append(fresh, drama)
}
}
}
return fresh
}
func (a *UIApp) enqueueHistoricalMetadataLocked(source string, priority []string) {
var available []Drama
for _, drama := range a.dramas {
if !a.metadataPending[drama.ID] {
available = append(available, drama)
}
}
a.enqueueSortMetadataLocked(selectSortMetadataBatch(available, source, priority, time.Now()), true)
}
func (a *UIApp) enqueueSortMetadataLocked(items []Drama, prioritize bool) {
if a.metadataClosed || len(items) == 0 {
return
}
if a.metadataPending == nil {
a.metadataPending = map[string]bool{}
}
var added []string
for _, drama := range items {
if drama.ID == "" || !needsSortMetadata(drama) || a.metadataPending[drama.ID] {
continue
}
a.metadataPending[drama.ID] = true
added = append(added, drama.ID)
}
if len(added) == 0 {
return
}
for index, drama := range a.dramas {
if !a.metadataPending[drama.ID] || drama.SortMetadata != nil && drama.SortMetadata.Pending {
continue
}
state := sortMetadataState{Pending: true}
if drama.SortMetadata != nil {
state = *drama.SortMetadata
state.Pending = true
}
a.dramas[index].SortMetadata = &state
}
if prioritize {
a.metadataQueue = append(added, a.metadataQueue...)
} else {
a.metadataQueue = append(a.metadataQueue, added...)
}
if !a.libraryMetadata.Running {
a.libraryMetadata = libraryMetadataProgress{Running: true}
}
a.libraryMetadata.Total += len(added)
a.libraryDirty = true
a.libraryRevision++
if a.metadataDone == nil {
ctx, cancel := context.WithCancel(context.Background())
a.metadataCancel = cancel
a.metadataDone = make(chan struct{})
go a.runSortMetadataQueue(ctx, a.metadataDone)
}
}
func (a *UIApp) runSortMetadataQueue(ctx context.Context, done chan struct{}) {
ctx = context.WithValue(ctx, backgroundCatalogKey{}, true)
for {
a.mu.Lock()
if ctx.Err() != nil || len(a.metadataQueue) == 0 {
a.mu.Unlock()
a.persistLibrary()
a.mu.Lock()
if ctx.Err() == nil && len(a.metadataQueue) > 0 {
a.mu.Unlock()
continue
}
a.libraryMetadata.Running = false
a.libraryRevision++
a.metadataCancel()
a.metadataCancel = nil
a.metadataDone = nil
a.metadataQueue = nil
close(done)
a.mu.Unlock()
return
}
if a.libraryLoading != nil {
a.mu.Unlock()
timer := time.NewTimer(500 * time.Millisecond)
select {
case <-ctx.Done():
case <-timer.C:
}
timer.Stop()
continue
}
id := a.metadataQueue[0]
a.metadataQueue = a.metadataQueue[1:]
var previous Drama
for _, drama := range a.dramas {
if drama.ID == id {
previous = drama
break
}
}
a.mu.Unlock()
itemCtx, cancel := context.WithTimeout(ctx, 40*time.Second)
patch, err := a.downloader.fetchDramaSortMetadata(itemCtx, previous)
cancel()
if err != nil && ctx.Err() == nil {
a.downloader.recordDiagnostic(diagnosticEvent{Event: "metadata.failed", Source: dramaProvider(previous), DramaID: previous.ID, DramaTitle: previous.DisplayTitle(), Message: err.Error()})
}
patch.SortMetadata = &sortMetadataState{CheckedAt: time.Now()}
if err == nil {
patch.SortMetadata.Version = sortMetadataVersion
patch.SortMetadata.CoverChecked = true
}
normalizeDramaCover(&patch)
a.mu.Lock()
if ctx.Err() != nil {
a.mu.Unlock()
continue
}
delete(a.metadataPending, id)
for index, drama := range a.dramas {
if drama.ID != id {
continue
}
if patch.OnlineDate != "" && patch.OnlineDate != drama.OnlineDate || patch.Heat != "" && patch.Heat != drama.Heat || patch.Views != "" && patch.Views != drama.Views || bestDramaCover(patch) != "" && bestDramaCover(patch) != bestDramaCover(drama) {
a.libraryMetadata.Updated++
}
patch.Title, patch.Name = drama.Title, drama.Name
a.dramas[index] = mergeDramaMetadata(patch, drama)
break
}
a.libraryMetadata.Checked++
if err != nil {
a.libraryMetadata.Failed++
}
a.libraryDirty = true
a.libraryRevision++
persist := time.Since(a.libraryLastSave) >= 3*time.Second
if persist {
a.libraryLastSave = time.Now()
}
a.mu.Unlock()
if persist {
a.persistLibrary()
}
}
}
func (a *UIApp) stopSortMetadata() {
a.mu.Lock()
a.metadataClosed = true
cancel, done := a.metadataCancel, a.metadataDone
a.mu.Unlock()
if cancel != nil {
cancel()
<-done
}
}
+243
View File
@@ -0,0 +1,243 @@
package app
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"sync/atomic"
"testing"
"time"
)
func waitSortMetadataQueue(t *testing.T, app *UIApp) {
t.Helper()
app.mu.Lock()
done := app.metadataDone
app.mu.Unlock()
if done != nil {
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("metadata queue did not finish")
}
}
}
func TestPrimaryCatalogReturnsBeforeOptionalMetadata(t *testing.T) {
for _, source := range []string{sourceHongguo} {
t.Run(source, func(t *testing.T) {
id := "7615465407347952664"
started, release := make(chan struct{}), make(chan struct{})
var once sync.Once
var calls atomic.Int32
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
calls.Add(1)
if request.Context().Value(backgroundCatalogKey{}) != true {
t.Error("optional detail did not use background priority")
}
once.Do(func() { close(started) })
select {
case <-release:
case <-request.Context().Done():
return nil, request.Context().Err()
}
switch request.URL.Path {
case "/detail":
if request.URL.Query().Get("series_id") != id {
t.Error("historical entry was automatically fetched")
}
return rankingHTTPResponse(request, 200, sortDetailFixture(id, "1773662280")), nil
case "/novel/player/video_detail/v1/":
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":%q,"series_title":"详情标题","hot_score":20019,"series_play_cnt":10019}}}`, id)), nil
default:
t.Errorf("unexpected metadata request: %s", request.URL.Path)
return rankingHTTPResponse(request, 404, ""), nil
}
})
old := Drama{ID: providerDramaID(source, "7498612570866076734"), Source: source, Title: "历史条目"}
fresh := Drama{ID: providerDramaID(source, id), Source: source, Title: "列表标题", Heat: "20019", Views: "0", Cover: coverAddressFixture}
app := &UIApp{downloader: d, cfg: d.cfg, dramas: []Drama{old}, librarySources: map[string]librarySourceState{}, libraryLoading: make(chan struct{})}
t.Cleanup(app.stopSortMetadata)
app.acceptLibraryProgress(source, []Drama{old, fresh}, nil, false)
app.mu.Lock()
if calls.Load() != 0 || len(app.dramas) != 2 || app.dramas[1].Heat != fresh.Heat || app.dramas[1].Views != "0" {
t.Error("primary fields did not return before optional requests")
}
close(app.libraryLoading)
app.libraryLoading = nil
app.mu.Unlock()
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("new entry was not scheduled automatically")
}
app.acceptLibraryProgress(source, []Drama{fresh, old}, nil, true)
writer := httptest.NewRecorder()
app.handleDramas(writer, httptest.NewRequest("GET", "/api/ui/dramas", nil))
var snapshot struct {
Loading bool `json:"loading"`
Metadata libraryMetadataProgress `json:"metadata"`
}
if json.Unmarshal(writer.Body.Bytes(), &snapshot) != nil || snapshot.Loading || !snapshot.Metadata.Running || snapshot.Metadata.Total != 1 {
t.Fatal("optional metadata blocked the library or duplicated work")
}
close(release)
waitSortMetadataQueue(t, app)
cache, err := readLibraryCache(d.cfg.dataDirectory())
if err != nil {
t.Fatal(err)
}
for _, row := range cache.Dramas {
if row.ID == old.ID && row.SortMetadata != nil {
t.Error("ordinary catalog refresh enrolled historical data")
}
wantViews := "10019次播放"
if source == sourceHongguo {
wantViews = "0"
}
if row.ID == fresh.ID && (row.OnlineDate != "2026-03-16" || row.Views != wantViews || row.Heat != fresh.Heat || row.Title != "列表标题" || row.SortMetadata == nil || row.SortMetadata.Version != sortMetadataVersion || row.SortMetadata.Pending) {
t.Errorf("asynchronous metadata was not merged correctly: %+v", row)
}
}
if calls.Load() != 1 {
t.Fatalf("duplicate detail requests: %d", calls.Load())
}
})
}
}
func TestResumeOnlyPreviouslyQueuedMetadata(t *testing.T) {
var calls atomic.Int32
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
calls.Add(1)
if request.URL.Path != "/detail" || request.URL.Query().Get("series_id") != "7000000000000000002" {
t.Errorf("unexpected resource: %s", request.URL.Path)
}
return rankingHTTPResponse(request, 200, sortDetailFixture("7000000000000000002", "0")), nil
})
cache := libraryCache{Dramas: []Drama{
{ID: "hongguo:7000000000000000001", Source: sourceHongguo, Title: "历史条目"},
{ID: "hongguo:7000000000000000002", Source: sourceHongguo, Title: "新条目", Heat: "0", Views: "0", Cover: coverAddressFixture, SortMetadata: &sortMetadataState{Pending: true}},
}}
if err := writeLibraryCache(d.cfg.dataDirectory(), cache); err != nil {
t.Fatal(err)
}
app := &UIApp{downloader: d, cfg: d.cfg}
t.Cleanup(app.stopSortMetadata)
app.loadLibrary()
waitSortMetadataQueue(t, app)
if calls.Load() != 1 {
t.Fatalf("resume requested %d resources, want one pending detail", calls.Load())
}
app.mu.Lock()
defer app.mu.Unlock()
if app.dramas[0].SortMetadata != nil || app.dramas[1].OnlineDate != "" || app.dramas[1].Views != "0" || app.dramas[1].SortMetadata.Version != sortMetadataVersion {
t.Fatal("resume changed history, lost zero, or invented an empty source date")
}
}
func TestPrimarySortFieldsNeedNoAdditionalInterfacesWhenComplete(t *testing.T) {
hongguo := hongguoDramaFromAny(map[string]any{"series_id": "7615465407347952664", "series_title": "列表剧", "series_cover": coverAddressFixture, "first_visible_time": "1773662280", "hot_score": "20019", "series_play_cnt": "0"}, "测试")
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
t.Errorf("complete primary data triggered an optional request: %s", request.URL.Path)
return rankingHTTPResponse(request, 404, ""), nil
})
for _, drama := range []Drama{hongguo} {
if drama.OnlineDate != "2026-03-16" || drama.Views != "0" || !hasCompleteSortMetadata(drama) {
t.Errorf("primary sort fields were lost: %+v", drama)
}
if _, err := d.fetchDramaSortMetadata(context.Background(), drama); err != nil {
t.Fatal(err)
}
}
if hasSortMetric("未知") || hasSortMetric("-1") {
t.Fatal("unknown metrics were misclassified as available sorting data")
}
}
func TestBackgroundRequestsYieldToForeground(t *testing.T) {
limiter := newRequestLimiter(1, time.Nanosecond)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
request := httptest.NewRequest("GET", "https://hongguoduanju.com/search/test", nil)
release, err := limiter.acquire(ctx, request)
if err != nil {
t.Fatal(err)
}
background := make(chan func(), 1)
foreground := make(chan func(), 1)
go func() {
unlock, _ := limiter.acquire(context.WithValue(ctx, backgroundCatalogKey{}, true), request)
background <- unlock
}()
go func() {
unlock, _ := limiter.acquire(ctx, request)
foreground <- unlock
}()
for {
limiter.mu.Lock()
waiting := limiter.foregroundWaiting
limiter.mu.Unlock()
if waiting > 0 {
break
}
if ctx.Err() != nil {
t.Fatal("foreground waiter did not register")
}
time.Sleep(time.Millisecond)
}
release()
select {
case unlock := <-foreground:
if unlock == nil {
t.Fatal("foreground timed out")
}
select {
case <-background:
t.Fatal("background ran alongside a waiting foreground request")
default:
}
unlock()
case <-background:
t.Fatal("optional metadata overtook foreground work")
case <-ctx.Done():
t.Fatal(ctx.Err())
}
select {
case unlock := <-background:
if unlock == nil {
t.Fatal("background did not resume")
}
unlock()
case <-ctx.Done():
t.Fatal(ctx.Err())
}
}
func TestOptionalDetailPermissionFailureDoesNotBlockForeground(t *testing.T) {
limiter := newRequestLimiter(1, time.Nanosecond)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
foreground := httptest.NewRequest("GET", "https://hongguoduanju.com/search/test", nil).WithContext(ctx)
background := foreground.WithContext(context.WithValue(ctx, backgroundCatalogKey{}, true))
limiter.observe(background, &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}})
if release, err := limiter.acquire(background.Context(), background); err == nil {
release()
t.Fatal("repeated forbidden details must cool down")
}
release, err := limiter.acquire(ctx, foreground)
if err != nil {
t.Fatal("an optional detail's permissions blocked public search", err)
}
release()
limiter.observe(background, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": {"60"}}})
if release, err := limiter.acquire(ctx, foreground); err == nil {
release()
t.Fatal("site rate limits must still be respected")
}
}
+185
View File
@@ -0,0 +1,185 @@
package app
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
)
func sortDetailFixture(id, stamp string) string {
return `<script>window._ROUTER_DATA={"loaderData":{"detail_page":{"seriesDetail":{"series_id":"` + id + `","series_name":"测试剧","first_visible_time":"` + stamp + `","create_time":"1745906791"}}}}</script>`
}
func TestSortDetailDatesAndIdentity(t *testing.T) {
id := "7615465407347952664"
drama, err := parseHongguoSortDetail(sortDetailFixture(id, "1773662280"), id)
if err != nil || drama.OnlineDate != "2026-03-16" {
t.Fatalf("first-visible date: %q, %v", drama.OnlineDate, err)
}
if _, err = parseHongguoSortDetail(sortDetailFixture("7498612570866076734", "1773662280"), id); err == nil {
t.Fatal("accepted another drama's time")
}
drama, err = parseHongguoSortDetail(sortDetailFixture(id, "0"), id)
if err != nil || drama.OnlineDate != "" {
t.Fatal("creation time must not replace missing first-visible time")
}
for _, value := range []string{"", "0", "0001-01-01", "1970-01-01", "2101-01-01", "2026-02-31", "not a date"} {
if got := providerReleaseDate(value); got != "" {
t.Errorf("accepted date %q", value)
}
}
if normalizeViews("10019") != "10019次播放" {
t.Fatal("play counts must retain precision")
}
}
func TestSortMetadataSelectionAndResume(t *testing.T) {
now := time.Now()
var rows []Drama
for index := 0; index < 60; index++ {
rows = append(rows, Drama{ID: fmt.Sprintf("hongguo:%019d", index+1), Source: sourceHongguo})
}
rows[1].SortMetadata = &sortMetadataState{Version: sortMetadataVersion, CheckedAt: now, CoverChecked: true}
rows[2].SortMetadata = &sortMetadataState{CheckedAt: now}
rows = append(rows, Drama{ID: "huangdou:other", Source: "huangdou"})
batch := selectSortMetadataBatch(rows, sourceHongguo, []string{rows[59].ID, rows[59].ID, rows[2].ID, "huangdou:other"}, now)
if len(batch) != sortMetadataBatchSize || batch[0].ID != rows[59].ID {
t.Fatal("did not prioritize filtered entries within the batch limit")
}
for _, row := range batch {
if row.ID == rows[1].ID || row.ID == rows[2].ID || row.Source != sourceHongguo {
t.Fatal("selected completed, cooling-down or other-source entries")
}
}
if got := selectSortMetadataBatch(rows, "huangdou", nil, now); len(got) != 0 {
t.Fatal("source filter was not respected")
}
if remaining := sortMetadataRemaining(rows); remaining["hongguo"] != 59 || remaining["huangdou"] != 0 || remaining[""] != 59 {
t.Fatal(remaining)
}
}
func TestManualBackfillPersistsPartialResultsAndContinuesAfterCatalogEnd(t *testing.T) {
first, bad, unknown := "7615465407347952664", "7498612570866076734", "7677914379060251673"
var calls atomic.Int32
var fail atomic.Bool
fail.Store(true)
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
calls.Add(1)
switch request.URL.Path {
case "/detail":
id := request.URL.Query().Get("series_id")
stamp := "1773662280"
if id == unknown {
stamp = "0"
}
return rankingHTTPResponse(request, 200, sortDetailFixture(id, stamp)), nil
case "/novel/player/video_detail/v1/":
var body struct {
ID string `json:"series_id"`
}
json.NewDecoder(request.Body).Decode(&body)
if body.ID == bad && fail.Load() {
return rankingHTTPResponse(request, 503, "temporary failure"), nil
}
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":%q,"series_title":"测试剧","hot_score":20019,"series_play_cnt":10019}}}`, body.ID)), nil
default:
t.Errorf("backfill requested a non-detail resource: %s", request.URL.Path)
return rankingHTTPResponse(request, 404, ""), nil
}
})
client := d.hongguoClient()
client.mu.Lock()
for _, genre := range hongguoAppGenres {
client.state.Feeds[genre.key] = hongguoCatalogCursor{Initialized: true, Exhausted: true}
}
client.mu.Unlock()
a := &UIApp{downloader: d, cfg: d.cfg, libraryAttempted: true, librarySources: map[string]librarySourceState{}, libraryApp: d.hongguoCatalogSnapshot(), dramas: []Drama{
{ID: "hongguo:" + first, Source: sourceHongguo, Title: "原有标题", Views: "0"},
{ID: "hongguo:" + bad, Source: sourceHongguo, Title: "暂时失败", Heat: "800"},
{ID: "hongguo:" + unknown, Source: sourceHongguo, Title: "站点日期未知"},
}}
t.Cleanup(a.stopSortMetadata)
plain := httptest.NewRecorder()
a.handleDramas(plain, httptest.NewRequest("GET", "/api/ui/dramas", nil))
if calls.Load() != 0 {
t.Fatal("ordinary library reads must not backfill")
}
before := a.librarySnapshotLocked(0)
if before["hasMoreBySource"].(map[string]bool)[sourceHongguo] != true {
t.Fatal("backfill must remain available after the catalog is exhausted")
}
writer := httptest.NewRecorder()
a.handleDramas(writer, httptest.NewRequest("GET", "/api/ui/dramas?more=1&source=hongguo&priority=hongguo:"+first, nil))
if writer.Code != 200 {
t.Fatal(writer.Body.String())
}
waitSortMetadataQueue(t, a)
a.mu.Lock()
rows := append([]Drama(nil), a.dramas...)
progress := a.libraryMetadata
a.mu.Unlock()
if progress.Checked != 3 || progress.Failed != 1 {
t.Fatalf("unexpected progress: %+v", progress)
}
for _, row := range rows {
switch row.ID {
case "hongguo:" + first:
if row.OnlineDate != "2026-03-16" || row.Heat != "20019" || row.Views != "10019次播放" || row.SortMetadata.Version != sortMetadataVersion {
t.Fatal("fresh sorting fields were not merged")
}
case "hongguo:" + bad:
if row.OnlineDate == "" || row.Heat != "800" || row.SortMetadata.Version != 0 {
t.Fatal("partial success must survive a failed metrics request")
}
case "hongguo:" + unknown:
if row.OnlineDate != "" || row.SortMetadata.Version != sortMetadataVersion {
t.Fatal("a valid response with no date must not be retried forever or invent a date")
}
}
}
saved, err := readLibraryCache(d.cfg.dataDirectory())
if err != nil {
t.Fatal(err)
}
if batch := selectSortMetadataBatch(saved.Dramas, sourceHongguo, nil, time.Now()); len(batch) != 0 {
t.Fatal("restart lost completed checks or failure cooldown")
}
for index, row := range saved.Dramas {
if row.ID == "hongguo:"+bad {
saved.Dramas[index].SortMetadata = &sortMetadataState{CheckedAt: time.Now().Add(-sortMetadataRetryDelay - time.Second)}
}
}
fail.Store(false)
previous := calls.Load()
patches, failures := d.backfillSortMetadata(withKnownHongguoDramas(context.Background(), saved.Dramas), sourceHongguo)
if len(failures) != 0 || len(patches) != 1 || patches[0].ID != "hongguo:"+bad || calls.Load()-previous != 1 {
t.Fatal("retry did not resume only the failed entry")
}
}
func TestManualMetadataPriorityValidation(t *testing.T) {
a := &UIApp{}
for _, path := range []string{
"/api/ui/dramas?more=1&source=invalid",
"/api/ui/dramas?more=1&refresh=1",
"/api/ui/dramas?update=1&refresh=1",
"/api/ui/dramas?update=1&more=1",
"/api/ui/dramas?update=1&source=invalid",
"/api/ui/dramas?update=1&priority=" + strings.Repeat("hongguo:700001,", sortMetadataBatchSize),
"/api/ui/dramas?update=1&priority=hongguo:" + strings.Repeat("1", 121),
"/api/ui/dramas?more=1&priority=" + strings.Repeat("id,", sortMetadataBatchSize),
} {
writer := httptest.NewRecorder()
a.handleDramas(writer, httptest.NewRequest("GET", path, nil))
if writer.Code != 400 {
t.Fatalf("invalid request was not rejected: %d", writer.Code)
}
}
}
+371
View File
@@ -0,0 +1,371 @@
package app
import (
"context"
"errors"
"fmt"
"net/http"
"sort"
"strings"
"time"
)
type librarySourceState struct {
Status string `json:"status"`
Count int `json:"count"`
Error string `json:"error,omitempty"`
Phase string `json:"phase,omitempty"`
UpdatedAt time.Time `json:"updatedAt"`
}
func (state librarySourceState) withPhase(phase string) librarySourceState {
phase = strings.TrimSpace(phase)
if state.Status == "loading" {
state.Phase = phase
}
return state
}
type libraryProgressKey struct{}
type libraryCategoryKey struct{}
type libraryProgressFunc func(string, []Drama, error, bool) error
type libraryLoadMode int
const (
libraryLoadRefresh libraryLoadMode = iota
libraryLoadMore
libraryLoadUpdate
)
func reportLibraryProgress(ctx context.Context, source string, items []Drama, err error, done bool) error {
if callback, ok := ctx.Value(libraryProgressKey{}).(libraryProgressFunc); ok && (done || len(items) > 0) {
return callback(source, items, err, done)
}
return nil
}
func (a *UIApp) setLibraryPhaseLocked(phase string) {
phase = strings.TrimSpace(phase)
a.libraryLoadingPhase = phase
if a.librarySources == nil {
a.librarySources = map[string]librarySourceState{}
}
for source, state := range a.librarySources {
if state.Status == "loading" {
a.librarySources[source] = state.withPhase(phase)
}
}
}
func (a *UIApp) handleLibraryCancel(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
return
}
a.mu.Lock()
cancel := a.libraryCancel
running := a.libraryLoading != nil
if running {
a.setLibraryPhaseLocked("正在停止,已保留已入库内容")
a.libraryRevision++
}
resp := a.librarySnapshotForSourceLocked(r.Context(), 0)
a.mu.Unlock()
if cancel != nil {
cancel()
}
resp["canceled"] = running
writeJSON(w, http.StatusOK, resp)
}
func (a *UIApp) startLibraryRefreshLocked(source string) {
a.startLibraryLoadLocked(source, libraryLoadRefresh, nil, "")
}
func (a *UIApp) startLibraryLoadLocked(source string, mode libraryLoadMode, priority []string, category string, scopes ...context.Context) {
category = strings.TrimSpace(category)
base := context.Background()
if len(scopes) > 0 && sourceScope(scopes[0]) != nil {
base = context.WithValue(base, accountSourceContextKey{}, sourceScope(scopes[0]))
}
if mode == libraryLoadUpdate || mode == libraryLoadMore {
if source == "" && sourceScopeRestricted(base) {
for _, allowed := range sourceScope(base).Sources {
a.enqueueHistoricalMetadataLocked(allowed, priority)
}
} else {
a.enqueueHistoricalMetadataLocked(source, priority)
}
}
more := mode == libraryLoadMore
if more {
if !matchesSourceFilter(sourceHongguo, source) || !hongguoCatalogCategoryHasMore(a.libraryApp, category) {
return
}
source = sourceHongguo
}
previousCount := len(a.dramas)
a.libraryAttempted = true
a.libraryLoading = make(chan struct{})
a.libraryMore = more
a.libraryLoadingSource = source
a.libraryLoadingPhase = "等待开始"
if a.librarySources == nil {
a.librarySources = map[string]librarySourceState{}
}
for _, provider := range []string{sourceHongguo} {
if matchesSourceFilter(provider, source) {
state := a.librarySources[provider]
state.Status = "loading"
state.Error = ""
state.Phase = "等待开始"
a.librarySources[provider] = state
}
}
a.libraryRevision++
ctx, cancel := context.WithTimeout(base, 12*time.Minute)
a.libraryCancel = cancel
ctx = context.WithValue(ctx, libraryProgressKey{}, libraryProgressFunc(a.acceptLibraryProgress))
ctx = context.WithValue(ctx, libraryMoreKey{}, more)
ctx = context.WithValue(ctx, libraryUpdateKey{}, mode == libraryLoadUpdate)
if category != "" {
ctx = context.WithValue(ctx, libraryCategoryKey{}, category)
}
ctx = withKnownHongguoDramas(ctx, a.dramas)
go func() {
defer cancel()
var loadErr error
a.mu.Lock()
a.setLibraryPhaseLocked("保存当前剧库")
a.libraryRevision++
a.mu.Unlock()
if err := a.persistLibraryIfDirty(); err != nil {
loadErr = err
} else if more {
a.mu.Lock()
a.setLibraryPhaseLocked("继续加载目录")
a.libraryRevision++
a.mu.Unlock()
items, err := a.downloader.fetchHongguoCategoryDramas(ctx, category)
loadErr = errors.Join(err, reportLibraryProgress(ctx, sourceHongguo, items, err, true))
} else {
a.mu.Lock()
a.setLibraryPhaseLocked("读取站源目录")
a.libraryRevision++
a.mu.Unlock()
_, loadErr = a.downloader.fetchAllDramas(ctx, source)
}
a.mu.Lock()
if ctx.Err() == context.Canceled {
loadErr = nil
a.markLibrarySourcesCanceledLocked(base, source)
} else if loadErr != nil && a.libraryLoading != nil {
a.markLibrarySourcesFailedLocked(base, source, loadErr)
}
a.libraryDirty = true
a.libraryRevision++
a.mu.Unlock()
_ = a.persistLibrary()
a.mu.Lock()
close(a.libraryLoading)
a.libraryLoading = nil
a.libraryCancel = nil
a.libraryLoadingPhase = ""
a.libraryRevision++
count := len(a.dramas)
a.mu.Unlock()
fmt.Printf("本地剧库:%d 部,本次新增 %d 部\n", count, count-previousCount)
}()
}
func (a *UIApp) persistLibraryIfDirty() error {
a.mu.Lock()
dirty := a.libraryDirty
a.mu.Unlock()
if !dirty {
return nil
}
return a.persistLibrary()
}
func (a *UIApp) markLibrarySourcesFailedLocked(ctx context.Context, source string, err error) {
if a.librarySources == nil {
a.librarySources = map[string]librarySourceState{}
}
if matchesSourceFilter(sourceHongguo, source) {
state := a.librarySources[sourceHongguo]
if state.Status == "loading" {
state.Status = "failed"
if state.Count > 0 {
state.Status = "partial"
}
state.Error = a.redactError(err)
state.Phase = ""
a.librarySources[sourceHongguo] = state
}
}
a.libraryError = librarySourceErrors(a.librarySources)
}
func (a *UIApp) markLibrarySourcesCanceledLocked(ctx context.Context, source string) {
if a.librarySources == nil {
a.librarySources = map[string]librarySourceState{}
}
for _, provider := range []string{sourceHongguo} {
if !matchesSourceFilter(provider, source) || !sourceAllowed(ctx, provider) {
continue
}
state := a.librarySources[provider]
if state.Status == "loading" {
state.Status = "canceled"
state.Error = ""
state.Phase = ""
a.librarySources[provider] = state
}
}
a.libraryError = librarySourceErrors(a.librarySources)
}
func (a *UIApp) acceptLibraryProgress(source string, items []Drama, loadErr error, done bool) error {
if canonicalProviderSource(source) != sourceHongguo {
return nil
}
source = sourceHongguo
a.mu.Lock()
items = onlyHongguoDramas(items)
a.normalizeDramaCovers(items)
newDramas := a.newSortMetadataDramasLocked(items)
if source == sourceHongguo {
a.libraryApp = a.downloader.hongguoCatalogSnapshot()
}
if done {
var mergeErr error
if loadErr != nil {
mergeErr = &libraryLoadError{failures: map[string]error{source: loadErr}}
}
a.dramas = mergeSourceDramas(a.dramas, items, mergeErr, source)
} else {
positions := make(map[string]int, len(a.dramas))
for index, drama := range a.dramas {
positions[drama.ID] = index
}
for _, drama := range items {
if drama.ID == "" {
continue
}
if position, found := positions[drama.ID]; found {
a.dramas[position] = mergeDramaMetadata(drama, a.dramas[position])
} else {
positions[drama.ID] = len(a.dramas)
a.dramas = append(a.dramas, drama)
}
}
}
a.enqueueSortMetadataLocked(newDramas, false)
state := a.librarySources[source]
state.Count = 0
for _, drama := range a.dramas {
if dramaProvider(drama) == source {
state.Count++
}
}
if len(items) > 0 {
a.loadedAt = time.Now()
state.UpdatedAt = a.loadedAt
}
if done {
state.Status = "ready"
if loadErr != nil {
state.Status = "failed"
if state.Count > 0 {
state.Status = "partial"
}
state.Error = a.redactError(loadErr)
}
state.Phase = ""
}
a.librarySources[source] = state
a.libraryError = librarySourceErrors(a.librarySources)
a.libraryDirty = true
a.libraryRevision++
persist := done || len(items) > 0 && source == sourceHongguo || time.Since(a.libraryLastSave) >= 3*time.Second
if persist {
a.libraryLastSave = time.Now()
}
a.mu.Unlock()
if persist {
return a.persistLibrary()
}
return nil
}
func librarySourceErrors(sources map[string]librarySourceState) string {
var messages []string
for source, state := range sources {
if state.Error != "" {
messages = append(messages, fmt.Sprintf("%s 获取失败: %s", source, state.Error))
}
}
sort.Strings(messages)
return strings.Join(messages, "\n")
}
func cloneLibrarySources(sources map[string]librarySourceState) map[string]librarySourceState {
copy := make(map[string]librarySourceState, len(sources))
for source, state := range sources {
copy[source] = state
}
return copy
}
func (a *UIApp) persistLibrary() error {
a.libraryWriteMu.Lock()
defer a.libraryWriteMu.Unlock()
a.mu.Lock()
cache := libraryCache{Dramas: append([]Drama{}, a.dramas...), LoadedAt: a.loadedAt, LastError: librarySourceErrors(a.librarySources), Sources: cloneLibrarySources(a.librarySources), HongguoApp: cloneHongguoCatalogState(a.libraryApp)}
revision := a.libraryRevision
a.mu.Unlock()
err := writeLibraryCache(a.cfg.dataDirectory(), cache)
a.mu.Lock()
defer a.mu.Unlock()
if err != nil {
a.libraryError = strings.TrimSpace(cache.LastError + "\n剧库缓存保存失败: " + a.redactError(err))
a.libraryRevision++
return err
}
a.librarySaved = true
if revision == a.libraryRevision {
a.libraryDirty = false
}
return nil
}
func (a *UIApp) librarySnapshotLocked(revision uint64) map[string]any {
remaining := sortMetadataRemaining(a.dramas)
var unqueued []Drama
for _, drama := range a.dramas {
if !a.metadataPending[drama.ID] {
unqueued = append(unqueued, drama)
}
}
more := map[string]bool{}
for source, count := range sortMetadataRemaining(unqueued) {
more[source] = count > 0
}
more["hongguo"] = more["hongguo"] || hongguoCatalogHasMore(a.libraryApp)
more[""] = more[""] || more["hongguo"]
response := map[string]any{
"revision": a.libraryRevision, "loadedAt": a.loadedAt, "loading": a.libraryLoading != nil,
"cached": a.librarySaved && !a.libraryDirty, "error": a.libraryError,
"sources": cloneLibrarySources(a.librarySources), "total": len(a.dramas),
"hasMore": more["hongguo"], "hasMoreBySource": more, "metadataRemaining": remaining,
"loadingMore": a.libraryMore && a.libraryLoading != nil, "loadingSource": a.libraryLoadingSource, "loadingPhase": a.libraryLoadingPhase, "metadata": a.libraryMetadata,
}
if revision != 0 && revision == a.libraryRevision {
response["unchanged"] = true
} else {
response["data"] = append([]Drama{}, a.dramas...)
}
return response
}
+322
View File
@@ -0,0 +1,322 @@
package app
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestUnifiedUpdateChecksNewContentAndContinuesCatalog(t *testing.T) {
for _, test := range []struct {
name string
pages int
savedOffset int
savedEnd bool
knownOffsets []int
endAt int
wantOffsets []int
wantSaved int
wantEnd bool
resumeSession bool
}{
{name: "first import runs one batch", pages: 2, endAt: -1, wantOffsets: []int{0, 18}, wantSaved: 36},
{name: "known head resumes saved tail", pages: 2, savedOffset: 90, knownOffsets: []int{0}, endAt: -1, wantOffsets: []int{0, 90, 108}, wantSaved: 126, resumeSession: true},
{name: "new head stops at overlap", pages: 4, savedOffset: 90, knownOffsets: []int{18}, endAt: -1, wantOffsets: []int{0, 18, 90, 108, 126, 144}, wantSaved: 162, resumeSession: true},
{name: "large update continues next unread page", pages: 4, savedOffset: 180, endAt: -1, wantOffsets: []int{0, 18, 36, 54, 72, 90, 108}, wantSaved: 126},
{name: "exhausted catalog still checks new content", pages: 4, savedOffset: 180, savedEnd: true, knownOffsets: []int{18}, endAt: -1, wantOffsets: []int{0, 18}, wantSaved: 180, wantEnd: true},
{name: "continuation reaches end", pages: 2, savedOffset: 90, knownOffsets: []int{0}, endAt: 90, wantOffsets: []int{0, 90}, wantSaved: 108, wantEnd: true, resumeSession: true},
} {
t.Run(test.name, func(t *testing.T) {
calls := map[string][]int{}
var known []Drama
for index := range hongguoAppGenres {
for _, offset := range test.knownOffsets {
known = append(known, Drama{ID: fmt.Sprintf("hongguo:%d", 790000+index*1000+offset), Source: sourceHongguo})
}
}
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
var payload struct {
Offset int `json:"offset"`
Session string `json:"session_id"`
Select struct {
Genre []string `json:"genre"`
} `json:"select_items"`
}
if request.URL.Path != "/reading/distribution/category/landpage/v/" || json.NewDecoder(request.Body).Decode(&payload) != nil || len(payload.Select.Genre) != 1 {
t.Errorf("unexpected catalog request: %s", request.URL.Path)
return rankingHTTPResponse(request, 400, ""), nil
}
genre := payload.Select.Genre[0]
index := -1
for position, candidate := range hongguoAppGenres {
if candidate.key == genre {
index = position
}
}
if index < 0 {
t.Errorf("unexpected genre: %s", genre)
return rankingHTTPResponse(request, 400, ""), nil
}
calls[genre] = append(calls[genre], payload.Offset)
if payload.Offset == 0 && payload.Session != "" {
t.Error("head check reused a tail session")
}
if test.resumeSession && payload.Offset == test.savedOffset && payload.Session != "saved-"+genre {
t.Error("continuation lost its saved session")
}
body, _ := json.Marshal(map[string]any{"data": map[string]any{
"next_offset": payload.Offset + 18, "session_id": "fixture-" + genre, "has_more": payload.Offset != test.endAt,
"video_data": []any{map[string]any{
"series_id": fmt.Sprint(790000 + index*1000 + payload.Offset), "series_title": "文字目录样本",
"first_visible_time": "1773662280", "hot_score": "20", "series_play_cnt": "0",
}},
}})
return rankingHTTPResponse(request, 200, string(body)), nil
})
d.cfg.MaxPagesPerSort = test.pages
if test.savedOffset > 0 {
client := d.hongguoClient()
for _, genre := range hongguoAppGenres {
client.state.Feeds[genre.key] = hongguoCatalogCursor{
Offset: test.savedOffset, SessionID: "saved-" + genre.key, Initialized: true, Exhausted: test.savedEnd, UpdatedAt: time.Now(),
}
}
}
ctx := withKnownHongguoDramas(context.WithValue(context.Background(), libraryUpdateKey{}, true), known)
items, err := d.fetchHongguoAppCatalog(ctx)
if err != nil || len(items) != len(test.wantOffsets)*len(hongguoAppGenres) {
t.Fatalf("catalog result: %d items, %v", len(items), err)
}
for _, genre := range hongguoAppGenres {
if !reflect.DeepEqual(calls[genre.key], test.wantOffsets) {
t.Errorf("%s requested %v, want %v", genre.key, calls[genre.key], test.wantOffsets)
}
cursor := d.hongguoCatalogSnapshot().Feeds[genre.key]
if !cursor.Initialized || cursor.Offset != test.wantSaved || cursor.Exhausted != test.wantEnd {
t.Errorf("wrong checkpoint for %s: %+v", genre.key, cursor)
}
}
})
}
}
func TestUnifiedUpdatePreservesHistoryAcrossRestart(t *testing.T) {
calls := map[string][]int{}
transport := func(request *http.Request) (*http.Response, error) {
var payload struct {
Offset int `json:"offset"`
Scene string `json:"req_scene"`
}
if request.URL.Path != "/reading/distribution/category/landpage/v/" || json.NewDecoder(request.Body).Decode(&payload) != nil {
t.Errorf("unexpected request: %s", request.URL.Path)
return rankingHTTPResponse(request, 400, ""), nil
}
calls[payload.Scene] = append(calls[payload.Scene], payload.Offset)
index := 0
for position, genre := range hongguoAppGenres {
if genre.scene == payload.Scene {
index = position
}
}
body := fmt.Sprintf(`{"data":{"next_offset":%d,"has_more":%t,"session_id":"saved-session","video_data":[{"series_id":"%d","series_title":"文字样本","first_visible_time":"1773662280","hot_score":"20","series_play_cnt":"0"}]}}`, payload.Offset+18, payload.Offset == 0, 790000+index*1000+payload.Offset)
return rankingHTTPResponse(request, 200, body), nil
}
directory := t.TempDir()
for run, wantCount := range []int{3, 6, 6} {
d := rankingTestDownloader(t, transport)
d.cfg.dataDir, d.cfg.MaxPagesPerSort = directory, 1
items, err := d.RefreshDramas(context.WithValue(context.Background(), libraryUpdateKey{}, true), sourceHongguo)
if err != nil || len(items) != wantCount {
t.Fatalf("run %d lost history: %d items, %v", run, len(items), err)
}
cache, err := readLibraryCache(directory)
if err != nil || len(cache.Dramas) != wantCount || hongguoCatalogHasMore(cache.HongguoApp) != (run == 0) {
t.Fatalf("run %d did not persist data and cursor together: %v", run, err)
}
for _, drama := range cache.Dramas {
if drama.OnlineDate != "2026-03-16" || drama.Heat != "20" || drama.Views != "0" {
t.Fatal("update lost available sorting fields")
}
}
}
for scene, offsets := range calls {
if !reflect.DeepEqual(offsets, []int{0, 0, 18, 0}) {
t.Errorf("%s skipped or repeated a continuation: %v", scene, offsets)
}
}
}
func TestUnifiedUpdateReturnsImmediatelyAndDeduplicatesWork(t *testing.T) {
catalogStarted, catalogRelease := make(chan struct{}), make(chan struct{})
detailStarted, detailRelease := make(chan string, 1), make(chan struct{})
releaseCatalog := sync.OnceFunc(func() { close(catalogRelease) })
releaseDetail := sync.OnceFunc(func() { close(detailRelease) })
var catalogOnce sync.Once
var catalogCalls atomic.Int32
var detailMu sync.Mutex
var detailIDs []string
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
switch request.URL.Path {
case "/reading/distribution/category/landpage/v/":
catalogCalls.Add(1)
catalogOnce.Do(func() { close(catalogStarted) })
select {
case <-catalogRelease:
case <-request.Context().Done():
return nil, request.Context().Err()
}
return rankingHTTPResponse(request, 200, `{"data":{"next_offset":18,"has_more":false,"video_data":[{"series_id":"790000","series_title":"已完成的目录条目","first_visible_time":"1773662280","hot_score":"20","series_play_cnt":"0"}]}}`), nil
case "/detail":
if request.Context().Value(backgroundCatalogKey{}) != true {
t.Error("historical metadata did not use background priority")
}
id := request.URL.Query().Get("series_id")
detailMu.Lock()
detailIDs = append(detailIDs, id)
if len(detailIDs) == 1 {
detailStarted <- id
}
detailMu.Unlock()
select {
case <-detailRelease:
case <-request.Context().Done():
return nil, request.Context().Err()
}
return rankingHTTPResponse(request, 200, sortDetailFixture(id, "1773662280")), nil
default:
t.Errorf("unexpected resource: %s", request.URL.Path)
return rankingHTTPResponse(request, 404, ""), nil
}
})
d.cfg.MaxPagesPerSort = 1
for _, genre := range hongguoAppGenres {
d.hongguoClient().state.Feeds[genre.key] = hongguoCatalogCursor{Initialized: true, Exhausted: true, Offset: 18}
}
app := &UIApp{downloader: d, cfg: d.cfg, libraryAttempted: true, libraryApp: d.hongguoCatalogSnapshot(), librarySources: map[string]librarySourceState{}}
app.dramas = append(app.dramas, Drama{ID: "hongguo:790000", Source: sourceHongguo, Title: "已完成的目录条目", OnlineDate: "2026-03-16", Heat: "20", Views: "0", Cover: coverAddressFixture, SortMetadata: &sortMetadataState{Version: sortMetadataVersion}})
for index := 0; index < 60; index++ {
app.dramas = append(app.dramas, Drama{ID: fmt.Sprintf("hongguo:%d", 800000+index), Source: sourceHongguo, Title: "历史文字样本", Heat: "20", Views: "0", Cover: coverAddressFixture})
}
t.Cleanup(func() {
releaseCatalog()
releaseDetail()
app.mu.Lock()
done, cancel := app.libraryLoading, app.libraryCancel
app.mu.Unlock()
if cancel != nil {
cancel()
}
if done != nil {
select {
case <-done:
case <-time.After(3 * time.Second):
t.Error("catalog worker did not stop")
}
}
app.stopSortMetadata()
})
type snapshot struct {
Accepted bool `json:"updateAccepted"`
Loading bool `json:"loading"`
Data []Drama `json:"data"`
Metadata libraryMetadataProgress `json:"metadata"`
}
read := func(path string) snapshot {
t.Helper()
done := make(chan *httptest.ResponseRecorder, 1)
go func() {
writer := httptest.NewRecorder()
app.handleDramas(writer, httptest.NewRequest("GET", path, nil))
done <- writer
}()
select {
case writer := <-done:
var result snapshot
if writer.Code != 200 || json.Unmarshal(writer.Body.Bytes(), &result) != nil {
t.Fatalf("invalid snapshot: %d %s", writer.Code, writer.Body.String())
}
return result
case <-time.After(2 * time.Second):
t.Fatal("library response waited for a stalled upstream request")
return snapshot{}
}
}
if result := read("/api/ui/dramas"); result.Loading || result.Metadata.Total != 0 || catalogCalls.Load() != 0 {
t.Fatal("ordinary cache read scheduled remote work")
}
path := "/api/ui/dramas?update=1&source=hongguo&priority=hongguo:800059"
result := read(path)
if !result.Accepted || !result.Loading || result.Metadata.Total != 40 || len(result.Data) != 61 {
t.Fatalf("update did not immediately return cache and bounded progress: %+v", result.Metadata)
}
select {
case <-catalogStarted:
case <-time.After(2 * time.Second):
t.Fatal("update did not start the main catalog")
}
for _, request := range []string{path, path, "/api/ui/dramas?revision=1"} {
if result := read(request); result.Accepted || !result.Loading || result.Metadata.Total != 40 {
t.Fatal("repeated update or polling duplicated work")
}
}
select {
case <-detailStarted:
t.Fatal("metadata did not yield to the catalog")
default:
}
app.mu.Lock()
catalogDone := app.libraryLoading
app.mu.Unlock()
releaseCatalog()
select {
case <-catalogDone:
case <-time.After(2 * time.Second):
t.Fatal("catalog waited for optional metadata")
}
select {
case id := <-detailStarted:
if id != "800059" {
t.Fatalf("current filtered result was not prioritized: %s", id)
}
case <-time.After(2 * time.Second):
t.Fatal("historical metadata was not scheduled")
}
if result := read("/api/ui/dramas"); result.Loading || !result.Metadata.Running || result.Metadata.Checked != 0 || catalogCalls.Load() != 3 {
t.Fatal("optional metadata blocked the main response or catalog work was duplicated")
}
releaseDetail()
waitSortMetadataQueue(t, app)
result = read("/api/ui/dramas")
if result.Metadata.Checked != 40 || result.Metadata.Failed != 0 || result.Metadata.Running {
t.Fatalf("unexpected completion: %+v", result.Metadata)
}
untouched := 0
for _, drama := range result.Data {
if strings.HasPrefix(drama.ID, "hongguo:800") && drama.SortMetadata == nil {
untouched++
}
}
if untouched != 20 {
t.Fatalf("update migrated more than one historical batch: %d untouched", untouched)
}
detailMu.Lock()
defer detailMu.Unlock()
seen := map[string]bool{}
for _, id := range detailIDs {
if seen[id] {
t.Fatalf("duplicate historical request: %s", id)
}
seen[id] = true
}
if len(seen) != 40 {
t.Fatalf("unexpected detail request count: %d", len(seen))
}
}
+21
View File
@@ -0,0 +1,21 @@
package app
import (
"errors"
)
func pkcs7Unpad(src []byte, blockSize int) ([]byte, error) {
if len(src) == 0 || len(src)%blockSize != 0 {
return nil, errors.New("invalid pkcs7 length")
}
pad := int(src[len(src)-1])
if pad == 0 || pad > blockSize || pad > len(src) {
return nil, errors.New("invalid pkcs7 padding")
}
for _, v := range src[len(src)-pad:] {
if int(v) != pad {
return nil, errors.New("invalid pkcs7 padding bytes")
}
}
return src[:len(src)-pad], nil
}