feat(app): add internal components batch 5
This commit is contained in:
@@ -0,0 +1,170 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
type libraryCache struct {
|
||||
Dramas []Drama `json:"dramas"`
|
||||
LoadedAt time.Time `json:"loadedAt"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
Sources map[string]librarySourceState `json:"sources,omitempty"`
|
||||
HongguoApp *hongguoCatalogState `json:"hongguoApp,omitempty"`
|
||||
}
|
||||
|
||||
func libraryCachePath(outputDir string) string {
|
||||
return filepath.Join(outputDir, "library.json")
|
||||
}
|
||||
|
||||
func readLibraryCache(outputDir string) (libraryCache, error) {
|
||||
var cache libraryCache
|
||||
path := libraryCachePath(outputDir)
|
||||
body, err := os.ReadFile(path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
path = filepath.Join(outputDir, "ui-state.json")
|
||||
body, err = os.ReadFile(path)
|
||||
}
|
||||
if err != nil {
|
||||
return cache, err
|
||||
}
|
||||
if err := json.Unmarshal(body, &cache); err != nil {
|
||||
return cache, fmt.Errorf("本地剧库缓存损坏 %s: %w", path, err)
|
||||
}
|
||||
originalCount := len(cache.Dramas)
|
||||
cache.Dramas = onlyHongguoDramas(cache.Dramas)
|
||||
sources := make(map[string]librarySourceState)
|
||||
for source, state := range cache.Sources {
|
||||
if canonicalProviderSource(source) == sourceHongguo {
|
||||
sources[sourceHongguo] = state
|
||||
}
|
||||
}
|
||||
if originalCount != len(cache.Dramas) || len(sources) != len(cache.Sources) {
|
||||
cache.LastError = librarySourceErrors(sources)
|
||||
}
|
||||
cache.Sources = sources
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func writeLibraryCache(outputDir string, cache libraryCache) error {
|
||||
if err := os.MkdirAll(outputDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
file, err := os.CreateTemp(outputDir, ".library-*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
encoder := json.NewEncoder(file)
|
||||
if err := encoder.Encode(cache); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if err := file.Sync(); err != nil {
|
||||
_ = file.Close()
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(file.Name(), libraryCachePath(outputDir))
|
||||
}
|
||||
|
||||
func (a *UIApp) loadLibrary() {
|
||||
cache, err := readLibraryCache(a.cfg.dataDirectory())
|
||||
if err == nil {
|
||||
a.downloader.restoreHongguoCatalog(cache.HongguoApp)
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrNotExist) {
|
||||
a.libraryError = a.redactError(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(cache.Dramas) > 0 && (len(a.dramas) == 0 || !a.loadedAt.After(cache.LoadedAt)) {
|
||||
a.dramas = cache.Dramas
|
||||
a.normalizeDramaCovers(a.dramas)
|
||||
a.loadedAt = cache.LoadedAt
|
||||
a.libraryError = cache.LastError
|
||||
}
|
||||
a.librarySources = cache.Sources
|
||||
if cache.HongguoApp != nil {
|
||||
a.libraryApp = a.downloader.hongguoCatalogSnapshot()
|
||||
}
|
||||
for source, state := range a.librarySources {
|
||||
if state.Status == "loading" {
|
||||
state.Status = "partial"
|
||||
state.Error = "上次更新中断,当前显示已保存数据,可手动刷新"
|
||||
a.librarySources[source] = state
|
||||
}
|
||||
}
|
||||
if sourceErrors := librarySourceErrors(a.librarySources); sourceErrors != "" {
|
||||
a.libraryError = sourceErrors
|
||||
}
|
||||
a.libraryRevision = 1
|
||||
if len(a.dramas) > 0 {
|
||||
if _, err := os.Stat(libraryCachePath(a.cfg.dataDirectory())); errors.Is(err, os.ErrNotExist) {
|
||||
if err := writeLibraryCache(a.cfg.dataDirectory(), libraryCache{Dramas: a.dramas, LoadedAt: a.loadedAt, LastError: a.libraryError, Sources: a.librarySources, HongguoApp: a.libraryApp}); err != nil {
|
||||
a.libraryError = "剧库缓存保存失败: " + a.redactError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
a.librarySaved = true
|
||||
var pending []Drama
|
||||
for _, drama := range a.dramas {
|
||||
if drama.SortMetadata != nil && drama.SortMetadata.Pending {
|
||||
pending = append(pending, drama)
|
||||
}
|
||||
}
|
||||
a.enqueueSortMetadataLocked(pending, false)
|
||||
fmt.Printf("已恢复本地剧库:%d 部,缓存:%s\n", len(a.dramas), libraryCachePath(a.cfg.dataDirectory()))
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) GetAllDramas(ctx context.Context) ([]Drama, error) {
|
||||
cache, err := readLibraryCache(d.cfg.dataDirectory())
|
||||
if err == nil {
|
||||
d.restoreHongguoCatalog(cache.HongguoApp)
|
||||
}
|
||||
if err == nil && len(cache.Dramas) > 0 {
|
||||
fmt.Printf("使用本地剧库缓存:%d 部(%s),更新请使用 -refresh\n", len(cache.Dramas), cache.LoadedAt.Format("2006-01-02 15:04:05"))
|
||||
if cache.LastError != "" {
|
||||
return cache.Dramas, errors.New(cache.LastError)
|
||||
}
|
||||
return cache.Dramas, nil
|
||||
}
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
fmt.Printf(" 读取本地剧库缓存失败: %v\n", err)
|
||||
}
|
||||
return d.RefreshDramas(ctx, "")
|
||||
}
|
||||
|
||||
func (d *Downloader) RefreshDramas(ctx context.Context, source string) ([]Drama, error) {
|
||||
cached, _ := readLibraryCache(d.cfg.dataDirectory())
|
||||
d.restoreHongguoCatalog(cached.HongguoApp)
|
||||
ctx = withKnownHongguoDramas(ctx, cached.Dramas)
|
||||
fresh, loadErr := d.fetchAllDramas(ctx, source)
|
||||
merged := mergeSourceDramas(cached.Dramas, fresh, loadErr, source)
|
||||
if len(fresh) == 0 && len(merged) == 0 {
|
||||
return nil, loadErr
|
||||
}
|
||||
cache := libraryCache{Dramas: merged, LoadedAt: time.Now(), HongguoApp: d.hongguoCatalogSnapshot()}
|
||||
if len(fresh) == 0 {
|
||||
cache.LoadedAt = cached.LoadedAt
|
||||
}
|
||||
if loadErr != nil {
|
||||
cache.LastError = publicError(loadErr).Error()
|
||||
}
|
||||
if err := writeLibraryCache(d.cfg.dataDirectory(), cache); err != nil {
|
||||
loadErr = errors.Join(loadErr, fmt.Errorf("剧库缓存保存失败: %w", err))
|
||||
}
|
||||
fmt.Printf("本地剧库:%d 部,本次新增 %d 部\n", len(merged), len(merged)-len(cached.Dramas))
|
||||
return merged, loadErr
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
const coverAddressFixture = "https://p3-reading-sign.fqnovelpic.com/synthetic-cover-address?fixture=1"
|
||||
|
||||
func TestHongguoRankingKeepsCoverAddressForLibrary(t *testing.T) {
|
||||
board, _ := findRankingBoard("hongguo-hot")
|
||||
rows := rankingRows(1)
|
||||
rows[0]["cover"] = coverAddressFixture
|
||||
page, err := parseHongguoRanking(rankingFixture(board, 1, rows), board, 1)
|
||||
if err != nil || bestDramaCover(page.Items[0].Drama) != coverAddressFixture || bestDramaCover(page.Items[1].Drama) != "" {
|
||||
t.Fatal("ranking discarded its cover address or invented one", err)
|
||||
}
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
t.Error("retaining an address must not fetch the image", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 404, ""), nil
|
||||
})
|
||||
app := &UIApp{downloader: d, cfg: d.cfg, metadataClosed: true}
|
||||
page.BoardID, page.Page, page.FetchedAt = board.ID, 1, time.Now()
|
||||
app.acceptRankingDramas(page)
|
||||
for _, drama := range app.dramas {
|
||||
if drama.ID == page.Items[0].Drama.ID && drama.Cover != "/api/ui/image?url="+url.QueryEscape(coverAddressFixture)+"&dramaId="+url.QueryEscape(drama.ID) {
|
||||
t.Fatal("ranking address bypassed the local cover route")
|
||||
}
|
||||
}
|
||||
for _, invalid := range []any{"javascript:invalid", "https://example.invalid/cover", "http://p3-reading-sign.fqnovelpic.com/cover", map[string]any{"unexpected": "field"}} {
|
||||
rows[0]["cover"] = invalid
|
||||
result, err := parseHongguoRanking(rankingFixture(board, 1, rows), board, 1)
|
||||
if err != nil || len(result.Items) != 2 || bestDramaCover(result.Items[0].Drama) != "" {
|
||||
t.Fatal("invalid optional address broke the ranking or reached the library", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistoricalMissingCoverWaitsForManualUpdate(t *testing.T) {
|
||||
for _, found := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("address=%t", found), func(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
calls.Add(1)
|
||||
if request.URL.Path != "/novel/player/video_detail/v1/" || request.Context().Value(backgroundCatalogKey{}) != true {
|
||||
t.Error("cover metadata used a non-background or non-text request", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 404, ""), nil
|
||||
}
|
||||
cover := ""
|
||||
if found {
|
||||
cover = coverAddressFixture
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":"7000000000000000001","series_title":"合成资料","series_cover":%q,"hot_score":999,"series_play_cnt":999}}}`, cover)), nil
|
||||
})
|
||||
old := Drama{ID: "hongguo:7000000000000000001", Source: sourceHongguo, Title: "历史榜单条目", OnlineDate: "2026-09-01", Heat: "20019", Views: "0",
|
||||
SortMetadata: &sortMetadataState{Version: sortMetadataVersion, CheckedAt: time.Now().Add(-time.Hour)}}
|
||||
if err := writeLibraryCache(d.cfg.dataDirectory(), libraryCache{Dramas: []Drama{old}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := &UIApp{downloader: d, cfg: d.cfg}
|
||||
t.Cleanup(app.stopSortMetadata)
|
||||
app.loadLibrary()
|
||||
app.handleDramas(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/ui/dramas", nil))
|
||||
app.mu.Lock()
|
||||
if calls.Load() != 0 || app.metadataDone != nil || len(app.newSortMetadataDramasLocked([]Drama{old})) != 0 || sortMetadataRemaining(app.dramas)[sourceHongguo] != 1 {
|
||||
app.mu.Unlock()
|
||||
t.Fatal("ordinary loading migrated historical addresses or hid their pending update")
|
||||
}
|
||||
app.enqueueHistoricalMetadataLocked(sourceHongguo, []string{old.ID})
|
||||
app.mu.Unlock()
|
||||
waitSortMetadataQueue(t, app)
|
||||
cache, err := readLibraryCache(d.cfg.dataDirectory())
|
||||
if err != nil || len(cache.Dramas) != 1 {
|
||||
t.Fatal("manual address repair was not persisted", err)
|
||||
}
|
||||
updated := cache.Dramas[0]
|
||||
if calls.Load() != 1 || updated.SortMetadata == nil || !updated.SortMetadata.CoverChecked || needsSortMetadata(updated) || updated.Views != "0" || updated.Heat != old.Heat || updated.Title != old.Title {
|
||||
t.Fatal("cover repair changed primary metadata or did not remember a completed check")
|
||||
}
|
||||
if found && (bestDramaCover(updated) != coverAddressFixture || !strings.HasPrefix(updated.Cover.(string), "/api/ui/image?url=")) || !found && bestDramaCover(updated) != "" {
|
||||
t.Fatal("manual repair lost the address or invented a missing cover")
|
||||
}
|
||||
app.mu.Lock()
|
||||
app.enqueueHistoricalMetadataLocked(sourceHongguo, []string{old.ID})
|
||||
app.mu.Unlock()
|
||||
waitSortMetadataQueue(t, app)
|
||||
if calls.Load() != 1 {
|
||||
t.Fatal("known missing cover triggered repeated requests")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHongguoCoverRepairValidatesDramaIdentity(t *testing.T) {
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":"7000000000000000002","series_cover":%q}}}`, coverAddressFixture)), nil
|
||||
})
|
||||
patch, err := d.fetchDramaSortMetadata(context.Background(), Drama{ID: "hongguo:7000000000000000001", OnlineDate: "2026-09-01", Heat: "1", Views: "0"})
|
||||
if err == nil || bestDramaCover(patch) != "" {
|
||||
t.Fatal("cover repair accepted another drama's address")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (d *Downloader) fetchAllDramas(ctx context.Context, sourceFilter string) ([]Drama, error) {
|
||||
if sourceFilter != "" && canonicalProviderSource(sourceFilter) != sourceHongguo {
|
||||
return nil, errors.New("此版本仅支持红果站源")
|
||||
}
|
||||
if more, _ := ctx.Value(libraryMoreKey{}).(bool); more {
|
||||
return d.fetchMoreLibrary(ctx, sourceFilter)
|
||||
}
|
||||
category, _ := ctx.Value(libraryCategoryKey{}).(string)
|
||||
fmt.Println("正在获取红果剧库列表...")
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
defer cancel()
|
||||
dramas, err := d.fetchHongguoCategoryDramas(ctx, category)
|
||||
dramas = onlyHongguoDramas(dramas)
|
||||
if len(dramas) == 0 && err == nil && !hongguoCatalogInitialized(d.hongguoCatalogSnapshot()) {
|
||||
err = errors.New("未返回可识别的视频数据")
|
||||
}
|
||||
reportLibraryProgress(ctx, sourceHongguo, dramas, err, true)
|
||||
fmt.Printf(" 剧库获取完成:%d 部\n", len(dramas))
|
||||
if err != nil {
|
||||
fmt.Printf(" 红果获取失败: %v\n", publicError(err))
|
||||
return dramas, &libraryLoadError{failures: map[string]error{sourceHongguo: err}}
|
||||
}
|
||||
return dramas, nil
|
||||
}
|
||||
|
||||
func (d *Downloader) GetDramaChapters(ctx context.Context, seriesID string) (string, []Chapter, error) {
|
||||
_, sourceID, valid := splitProviderDramaID(seriesID)
|
||||
if !valid {
|
||||
return "", nil, errors.New("仅支持红果剧集 ID,可填写 hongguo:数字ID 或数字ID")
|
||||
}
|
||||
title, chapters, err := d.fetchHongguoChapters(ctx, sourceID)
|
||||
return title, uniqueChapters(chapters), err
|
||||
}
|
||||
|
||||
func uniqueChapters(chapters []Chapter) []Chapter {
|
||||
seen := map[string]bool{}
|
||||
var out []Chapter
|
||||
for i, ch := range chapters {
|
||||
key := ch.ID
|
||||
if key == "" {
|
||||
key = ch.VideoURL
|
||||
}
|
||||
if key == "" {
|
||||
key = fmt.Sprintf("idx_%d", i)
|
||||
}
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
out = append(out, ch)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (d *Downloader) BuildDramaTasks(ctx context.Context, drama Drama) ([]Task, error) {
|
||||
return d.buildDramaTasksInDirectory(ctx, drama, "")
|
||||
}
|
||||
|
||||
func (d *Downloader) buildDramaTasksInDirectory(ctx context.Context, drama Drama, existingDirectory string) ([]Task, error) {
|
||||
var valid bool
|
||||
drama, valid = normalizeHongguoDrama(drama)
|
||||
if !valid {
|
||||
return nil, errors.New("此版本仅支持红果剧集")
|
||||
}
|
||||
title, chapters, err := d.GetDramaChapters(ctx, drama.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(chapters) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
drama = d.hongguoCachedDrama(drama)
|
||||
dramaDir, err := d.downloadDramaDirectory(drama, title, existingDirectory)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var tasks []Task
|
||||
for i, ch := range chapters {
|
||||
ep := ch.EpisodeString(i + 1)
|
||||
epNum := padEpisode(ep)
|
||||
fileName := safeFilename(epNum + ".mp4")
|
||||
outPath, err := safeJoin(dramaDir, fileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tasks = append(tasks, Task{DramaID: drama.ID, DramaTitle: title, Chapter: ch, Index: i + 1, Total: len(chapters), OutPath: outPath, ReleaseStatus: dramaReleaseStatus(drama)})
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const sortMetadataVersion = 1
|
||||
const sortMetadataBatchSize = 40
|
||||
const sortMetadataRetryDelay = 5 * time.Minute
|
||||
const sortMetadataMissingRetryDelay = 24 * time.Hour
|
||||
|
||||
type sortMetadataState struct {
|
||||
Version int `json:"version"`
|
||||
CheckedAt time.Time `json:"checkedAt"`
|
||||
Pending bool `json:"pending,omitempty"`
|
||||
CoverChecked bool `json:"coverChecked,omitempty"`
|
||||
RetryAt time.Time `json:"retryAt,omitempty"`
|
||||
LastError string `json:"lastError,omitempty"`
|
||||
}
|
||||
|
||||
type libraryRowsKey struct{}
|
||||
type libraryPriorityKey struct{}
|
||||
type libraryMetadataProgressKey struct{}
|
||||
|
||||
type libraryMetadataProgress struct {
|
||||
Running bool `json:"running"`
|
||||
Checked int `json:"checked"`
|
||||
Total int `json:"total"`
|
||||
Updated int `json:"updated"`
|
||||
Failed int `json:"failed"`
|
||||
}
|
||||
|
||||
func supportsSortMetadata(drama Drama) bool {
|
||||
switch dramaProvider(drama) {
|
||||
case sourceHongguo:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func needsSortMetadata(drama Drama) bool {
|
||||
return supportsSortMetadata(drama) && (drama.SortMetadata == nil || drama.SortMetadata.Version != sortMetadataVersion || needsHongguoCoverAddress(drama))
|
||||
}
|
||||
|
||||
func sortMetadataRetryAt(state *sortMetadataState, now time.Time) time.Time {
|
||||
if state == nil || state.Version != 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
if !state.RetryAt.IsZero() {
|
||||
return state.RetryAt
|
||||
}
|
||||
if state.CheckedAt.IsZero() {
|
||||
return time.Time{}
|
||||
}
|
||||
return state.CheckedAt.Add(sortMetadataRetryDelay)
|
||||
}
|
||||
|
||||
func sortMetadataFailureDelay(err error) time.Duration {
|
||||
var status *httpStatusError
|
||||
if errors.As(err, &status) && (status.status == http.StatusNotFound || status.status == http.StatusGone) {
|
||||
return sortMetadataMissingRetryDelay
|
||||
}
|
||||
text := strings.ToLower(err.Error())
|
||||
if strings.Contains(text, "无效的") || strings.Contains(text, "未返回所请求") || strings.Contains(text, "返回了其他剧集") || strings.Contains(text, "与请求剧集不符") {
|
||||
return sortMetadataMissingRetryDelay
|
||||
}
|
||||
return sortMetadataRetryDelay
|
||||
}
|
||||
|
||||
func failedSortMetadataState(drama Drama, err error, now time.Time) *sortMetadataState {
|
||||
state := &sortMetadataState{CheckedAt: now, RetryAt: now.Add(sortMetadataFailureDelay(err)), LastError: truncate(publicError(err).Error(), 300)}
|
||||
if drama.SortMetadata != nil {
|
||||
state.CoverChecked = drama.SortMetadata.CoverChecked
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
func needsHongguoCoverAddress(drama Drama) bool {
|
||||
return dramaProvider(drama) == sourceHongguo && bestDramaCover(drama) == "" &&
|
||||
(drama.SortMetadata == nil || !drama.SortMetadata.CoverChecked)
|
||||
}
|
||||
|
||||
func selectSortMetadataBatch(dramas []Drama, source string, priority []string, now time.Time) []Drama {
|
||||
positions := make(map[string]int, len(priority))
|
||||
for index, id := range priority {
|
||||
if _, found := positions[id]; !found {
|
||||
positions[id] = index
|
||||
}
|
||||
}
|
||||
var candidates []Drama
|
||||
seen := map[string]bool{}
|
||||
for _, drama := range dramas {
|
||||
if seen[drama.ID] || !matchesSourceFilter(dramaProvider(drama), source) || !needsSortMetadata(drama) {
|
||||
continue
|
||||
}
|
||||
seen[drama.ID] = true
|
||||
if retryAt := sortMetadataRetryAt(drama.SortMetadata, now); !retryAt.IsZero() && now.Before(retryAt) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, drama)
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
left, leftOK := positions[candidates[i].ID]
|
||||
right, rightOK := positions[candidates[j].ID]
|
||||
if leftOK != rightOK {
|
||||
return leftOK
|
||||
}
|
||||
if leftOK && left != right {
|
||||
return left < right
|
||||
}
|
||||
var leftTime, rightTime time.Time
|
||||
if candidates[i].SortMetadata != nil {
|
||||
leftTime = candidates[i].SortMetadata.CheckedAt
|
||||
}
|
||||
if candidates[j].SortMetadata != nil {
|
||||
rightTime = candidates[j].SortMetadata.CheckedAt
|
||||
}
|
||||
return leftTime.Before(rightTime)
|
||||
})
|
||||
if len(candidates) > sortMetadataBatchSize {
|
||||
candidates = candidates[:sortMetadataBatchSize]
|
||||
}
|
||||
return candidates
|
||||
}
|
||||
|
||||
func sortMetadataRemaining(dramas []Drama) map[string]int {
|
||||
counts := map[string]int{"": 0, "hongguo": 0}
|
||||
for _, drama := range dramas {
|
||||
if !needsSortMetadata(drama) {
|
||||
continue
|
||||
}
|
||||
source := dramaProvider(drama)
|
||||
counts[source]++
|
||||
counts[""]++
|
||||
}
|
||||
return counts
|
||||
}
|
||||
|
||||
func reportMetadataProgress(ctx context.Context, progress libraryMetadataProgress) {
|
||||
if callback, ok := ctx.Value(libraryMetadataProgressKey{}).(func(libraryMetadataProgress)); ok {
|
||||
callback(progress)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Downloader) backfillSortMetadata(ctx context.Context, source string) ([]Drama, map[string]error) {
|
||||
rows, _ := ctx.Value(libraryRowsKey{}).([]Drama)
|
||||
priority, _ := ctx.Value(libraryPriorityKey{}).([]string)
|
||||
batch := selectSortMetadataBatch(rows, source, priority, time.Now())
|
||||
progress := libraryMetadataProgress{Running: true, Total: len(batch)}
|
||||
reportMetadataProgress(ctx, progress)
|
||||
defer func() { progress.Running = false; reportMetadataProgress(ctx, progress) }()
|
||||
failures := map[string]error{}
|
||||
if len(batch) == 0 {
|
||||
return nil, failures
|
||||
}
|
||||
workCtx, cancel := context.WithTimeout(ctx, 3*time.Minute)
|
||||
defer cancel()
|
||||
type result struct {
|
||||
patch Drama
|
||||
previous Drama
|
||||
err error
|
||||
}
|
||||
jobs := make(chan Drama, len(batch))
|
||||
results := make(chan result, len(batch))
|
||||
for _, drama := range batch {
|
||||
jobs <- drama
|
||||
}
|
||||
close(jobs)
|
||||
workers := d.cfg.RequestConcurrency
|
||||
if workers < 1 {
|
||||
workers = 1
|
||||
}
|
||||
if workers > 3 {
|
||||
workers = 3
|
||||
}
|
||||
var group sync.WaitGroup
|
||||
for worker := 0; worker < workers; worker++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for drama := range jobs {
|
||||
if workCtx.Err() != nil {
|
||||
return
|
||||
}
|
||||
itemCtx, stop := context.WithTimeout(workCtx, 40*time.Second)
|
||||
patch, err := d.fetchDramaSortMetadata(itemCtx, drama)
|
||||
stop()
|
||||
now := time.Now()
|
||||
patch.SortMetadata = &sortMetadataState{CheckedAt: now}
|
||||
if err == nil {
|
||||
patch.SortMetadata.Version = sortMetadataVersion
|
||||
patch.SortMetadata.CoverChecked = true
|
||||
} else {
|
||||
patch.SortMetadata = failedSortMetadataState(drama, err, now)
|
||||
}
|
||||
results <- result{patch: patch, previous: drama, err: err}
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() { group.Wait(); close(results) }()
|
||||
var patches []Drama
|
||||
for result := range results {
|
||||
patch := result.patch
|
||||
provider := dramaProvider(result.previous)
|
||||
if result.err != nil {
|
||||
progress.Failed++
|
||||
if failures[provider] == nil {
|
||||
failures[provider] = fmt.Errorf("部分历史资料未补齐,可稍后重试: %w", result.err)
|
||||
}
|
||||
}
|
||||
if patch.OnlineDate != "" && patch.OnlineDate != result.previous.OnlineDate || patch.Heat != "" && patch.Heat != result.previous.Heat || patch.Views != "" && patch.Views != result.previous.Views || bestDramaCover(patch) != "" && bestDramaCover(patch) != bestDramaCover(result.previous) {
|
||||
progress.Updated++
|
||||
}
|
||||
patches = append(patches, patch)
|
||||
progress.Checked++
|
||||
reportLibraryProgress(ctx, provider, []Drama{patch}, nil, false)
|
||||
reportMetadataProgress(ctx, progress)
|
||||
}
|
||||
if progress.Checked < len(batch) {
|
||||
for _, drama := range batch {
|
||||
provider := dramaProvider(drama)
|
||||
if failures[provider] == nil {
|
||||
failures[provider] = errors.New("本批补齐已暂停,剩余条目可再次手动补齐")
|
||||
}
|
||||
}
|
||||
}
|
||||
return patches, failures
|
||||
}
|
||||
|
||||
func (d *Downloader) fetchMoreLibrary(ctx context.Context, source string) ([]Drama, error) {
|
||||
category, _ := ctx.Value(libraryCategoryKey{}).(string)
|
||||
patches, failures := d.backfillSortMetadata(ctx, source)
|
||||
if matchesSourceFilter(sourceHongguo, source) {
|
||||
items, err := d.fetchHongguoCategoryDramas(ctx, category)
|
||||
if err != nil {
|
||||
failures[sourceHongguo] = errors.Join(failures[sourceHongguo], err)
|
||||
}
|
||||
reportLibraryProgress(ctx, sourceHongguo, items, failures[sourceHongguo], true)
|
||||
patches = mergeSourceDramas(patches, items, nil, sourceHongguo)
|
||||
}
|
||||
if len(failures) > 0 {
|
||||
return patches, &libraryLoadError{failures: failures}
|
||||
}
|
||||
return patches, nil
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (a *UIApp) newSortMetadataDramasLocked(items []Drama) []Drama {
|
||||
known := make(map[string]bool, len(a.dramas))
|
||||
for _, drama := range a.dramas {
|
||||
known[drama.ID] = true
|
||||
}
|
||||
var fresh []Drama
|
||||
for _, drama := range items {
|
||||
if drama.ID != "" && !known[drama.ID] {
|
||||
known[drama.ID] = true
|
||||
if needsSortMetadata(drama) {
|
||||
fresh = append(fresh, drama)
|
||||
}
|
||||
}
|
||||
}
|
||||
return fresh
|
||||
}
|
||||
|
||||
func (a *UIApp) enqueueHistoricalMetadataLocked(source string, priority []string) {
|
||||
var available []Drama
|
||||
for _, drama := range a.dramas {
|
||||
if !a.metadataPending[drama.ID] {
|
||||
available = append(available, drama)
|
||||
}
|
||||
}
|
||||
a.enqueueSortMetadataLocked(selectSortMetadataBatch(available, source, priority, time.Now()), true)
|
||||
}
|
||||
|
||||
func (a *UIApp) enqueueSortMetadataLocked(items []Drama, prioritize bool) {
|
||||
if a.metadataClosed || len(items) == 0 {
|
||||
return
|
||||
}
|
||||
if a.metadataPending == nil {
|
||||
a.metadataPending = map[string]bool{}
|
||||
}
|
||||
var added []string
|
||||
for _, drama := range items {
|
||||
if drama.ID == "" || !needsSortMetadata(drama) || a.metadataPending[drama.ID] {
|
||||
continue
|
||||
}
|
||||
a.metadataPending[drama.ID] = true
|
||||
added = append(added, drama.ID)
|
||||
}
|
||||
if len(added) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
for index, drama := range a.dramas {
|
||||
if !a.metadataPending[drama.ID] || drama.SortMetadata != nil && drama.SortMetadata.Pending {
|
||||
continue
|
||||
}
|
||||
state := sortMetadataState{Pending: true}
|
||||
if drama.SortMetadata != nil {
|
||||
state = *drama.SortMetadata
|
||||
state.Pending = true
|
||||
}
|
||||
a.dramas[index].SortMetadata = &state
|
||||
}
|
||||
if prioritize {
|
||||
a.metadataQueue = append(added, a.metadataQueue...)
|
||||
} else {
|
||||
a.metadataQueue = append(a.metadataQueue, added...)
|
||||
}
|
||||
if !a.libraryMetadata.Running {
|
||||
a.libraryMetadata = libraryMetadataProgress{Running: true}
|
||||
}
|
||||
a.libraryMetadata.Total += len(added)
|
||||
a.libraryDirty = true
|
||||
a.libraryRevision++
|
||||
if a.metadataDone == nil {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
a.metadataCancel = cancel
|
||||
a.metadataDone = make(chan struct{})
|
||||
go a.runSortMetadataQueue(ctx, a.metadataDone)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *UIApp) runSortMetadataQueue(ctx context.Context, done chan struct{}) {
|
||||
|
||||
ctx = context.WithValue(ctx, backgroundCatalogKey{}, true)
|
||||
for {
|
||||
a.mu.Lock()
|
||||
if ctx.Err() != nil || len(a.metadataQueue) == 0 {
|
||||
a.mu.Unlock()
|
||||
a.persistLibrary()
|
||||
a.mu.Lock()
|
||||
|
||||
if ctx.Err() == nil && len(a.metadataQueue) > 0 {
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
a.libraryMetadata.Running = false
|
||||
a.libraryRevision++
|
||||
a.metadataCancel()
|
||||
a.metadataCancel = nil
|
||||
a.metadataDone = nil
|
||||
a.metadataQueue = nil
|
||||
close(done)
|
||||
a.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if a.libraryLoading != nil {
|
||||
a.mu.Unlock()
|
||||
timer := time.NewTimer(500 * time.Millisecond)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
timer.Stop()
|
||||
continue
|
||||
}
|
||||
id := a.metadataQueue[0]
|
||||
a.metadataQueue = a.metadataQueue[1:]
|
||||
var previous Drama
|
||||
for _, drama := range a.dramas {
|
||||
if drama.ID == id {
|
||||
previous = drama
|
||||
break
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
itemCtx, cancel := context.WithTimeout(ctx, 40*time.Second)
|
||||
patch, err := a.downloader.fetchDramaSortMetadata(itemCtx, previous)
|
||||
cancel()
|
||||
if err != nil && ctx.Err() == nil {
|
||||
a.downloader.recordDiagnostic(diagnosticEvent{Event: "metadata.failed", Source: dramaProvider(previous), DramaID: previous.ID, DramaTitle: previous.DisplayTitle(), Message: err.Error()})
|
||||
}
|
||||
patch.SortMetadata = &sortMetadataState{CheckedAt: time.Now()}
|
||||
if err == nil {
|
||||
patch.SortMetadata.Version = sortMetadataVersion
|
||||
patch.SortMetadata.CoverChecked = true
|
||||
}
|
||||
normalizeDramaCover(&patch)
|
||||
a.mu.Lock()
|
||||
if ctx.Err() != nil {
|
||||
|
||||
a.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
delete(a.metadataPending, id)
|
||||
for index, drama := range a.dramas {
|
||||
if drama.ID != id {
|
||||
continue
|
||||
}
|
||||
if patch.OnlineDate != "" && patch.OnlineDate != drama.OnlineDate || patch.Heat != "" && patch.Heat != drama.Heat || patch.Views != "" && patch.Views != drama.Views || bestDramaCover(patch) != "" && bestDramaCover(patch) != bestDramaCover(drama) {
|
||||
a.libraryMetadata.Updated++
|
||||
}
|
||||
|
||||
patch.Title, patch.Name = drama.Title, drama.Name
|
||||
a.dramas[index] = mergeDramaMetadata(patch, drama)
|
||||
break
|
||||
}
|
||||
a.libraryMetadata.Checked++
|
||||
if err != nil {
|
||||
a.libraryMetadata.Failed++
|
||||
}
|
||||
a.libraryDirty = true
|
||||
a.libraryRevision++
|
||||
persist := time.Since(a.libraryLastSave) >= 3*time.Second
|
||||
if persist {
|
||||
a.libraryLastSave = time.Now()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if persist {
|
||||
a.persistLibrary()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *UIApp) stopSortMetadata() {
|
||||
a.mu.Lock()
|
||||
a.metadataClosed = true
|
||||
cancel, done := a.metadataCancel, a.metadataDone
|
||||
a.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
<-done
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func waitSortMetadataQueue(t *testing.T, app *UIApp) {
|
||||
t.Helper()
|
||||
app.mu.Lock()
|
||||
done := app.metadataDone
|
||||
app.mu.Unlock()
|
||||
if done != nil {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("metadata queue did not finish")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrimaryCatalogReturnsBeforeOptionalMetadata(t *testing.T) {
|
||||
for _, source := range []string{sourceHongguo} {
|
||||
t.Run(source, func(t *testing.T) {
|
||||
id := "7615465407347952664"
|
||||
started, release := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
var calls atomic.Int32
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
calls.Add(1)
|
||||
if request.Context().Value(backgroundCatalogKey{}) != true {
|
||||
t.Error("optional detail did not use background priority")
|
||||
}
|
||||
once.Do(func() { close(started) })
|
||||
select {
|
||||
case <-release:
|
||||
case <-request.Context().Done():
|
||||
return nil, request.Context().Err()
|
||||
}
|
||||
switch request.URL.Path {
|
||||
case "/detail":
|
||||
if request.URL.Query().Get("series_id") != id {
|
||||
t.Error("historical entry was automatically fetched")
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, sortDetailFixture(id, "1773662280")), nil
|
||||
case "/novel/player/video_detail/v1/":
|
||||
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":%q,"series_title":"详情标题","hot_score":20019,"series_play_cnt":10019}}}`, id)), nil
|
||||
default:
|
||||
t.Errorf("unexpected metadata request: %s", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 404, ""), nil
|
||||
}
|
||||
})
|
||||
old := Drama{ID: providerDramaID(source, "7498612570866076734"), Source: source, Title: "历史条目"}
|
||||
fresh := Drama{ID: providerDramaID(source, id), Source: source, Title: "列表标题", Heat: "20019", Views: "0", Cover: coverAddressFixture}
|
||||
app := &UIApp{downloader: d, cfg: d.cfg, dramas: []Drama{old}, librarySources: map[string]librarySourceState{}, libraryLoading: make(chan struct{})}
|
||||
t.Cleanup(app.stopSortMetadata)
|
||||
app.acceptLibraryProgress(source, []Drama{old, fresh}, nil, false)
|
||||
app.mu.Lock()
|
||||
if calls.Load() != 0 || len(app.dramas) != 2 || app.dramas[1].Heat != fresh.Heat || app.dramas[1].Views != "0" {
|
||||
t.Error("primary fields did not return before optional requests")
|
||||
}
|
||||
close(app.libraryLoading)
|
||||
app.libraryLoading = nil
|
||||
app.mu.Unlock()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("new entry was not scheduled automatically")
|
||||
}
|
||||
|
||||
app.acceptLibraryProgress(source, []Drama{fresh, old}, nil, true)
|
||||
writer := httptest.NewRecorder()
|
||||
app.handleDramas(writer, httptest.NewRequest("GET", "/api/ui/dramas", nil))
|
||||
var snapshot struct {
|
||||
Loading bool `json:"loading"`
|
||||
Metadata libraryMetadataProgress `json:"metadata"`
|
||||
}
|
||||
if json.Unmarshal(writer.Body.Bytes(), &snapshot) != nil || snapshot.Loading || !snapshot.Metadata.Running || snapshot.Metadata.Total != 1 {
|
||||
t.Fatal("optional metadata blocked the library or duplicated work")
|
||||
}
|
||||
close(release)
|
||||
waitSortMetadataQueue(t, app)
|
||||
cache, err := readLibraryCache(d.cfg.dataDirectory())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, row := range cache.Dramas {
|
||||
if row.ID == old.ID && row.SortMetadata != nil {
|
||||
t.Error("ordinary catalog refresh enrolled historical data")
|
||||
}
|
||||
wantViews := "10019次播放"
|
||||
if source == sourceHongguo {
|
||||
wantViews = "0"
|
||||
}
|
||||
if row.ID == fresh.ID && (row.OnlineDate != "2026-03-16" || row.Views != wantViews || row.Heat != fresh.Heat || row.Title != "列表标题" || row.SortMetadata == nil || row.SortMetadata.Version != sortMetadataVersion || row.SortMetadata.Pending) {
|
||||
t.Errorf("asynchronous metadata was not merged correctly: %+v", row)
|
||||
}
|
||||
}
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("duplicate detail requests: %d", calls.Load())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeOnlyPreviouslyQueuedMetadata(t *testing.T) {
|
||||
var calls atomic.Int32
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
calls.Add(1)
|
||||
if request.URL.Path != "/detail" || request.URL.Query().Get("series_id") != "7000000000000000002" {
|
||||
t.Errorf("unexpected resource: %s", request.URL.Path)
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, sortDetailFixture("7000000000000000002", "0")), nil
|
||||
})
|
||||
cache := libraryCache{Dramas: []Drama{
|
||||
{ID: "hongguo:7000000000000000001", Source: sourceHongguo, Title: "历史条目"},
|
||||
{ID: "hongguo:7000000000000000002", Source: sourceHongguo, Title: "新条目", Heat: "0", Views: "0", Cover: coverAddressFixture, SortMetadata: &sortMetadataState{Pending: true}},
|
||||
}}
|
||||
if err := writeLibraryCache(d.cfg.dataDirectory(), cache); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
app := &UIApp{downloader: d, cfg: d.cfg}
|
||||
t.Cleanup(app.stopSortMetadata)
|
||||
app.loadLibrary()
|
||||
waitSortMetadataQueue(t, app)
|
||||
if calls.Load() != 1 {
|
||||
t.Fatalf("resume requested %d resources, want one pending detail", calls.Load())
|
||||
}
|
||||
app.mu.Lock()
|
||||
defer app.mu.Unlock()
|
||||
if app.dramas[0].SortMetadata != nil || app.dramas[1].OnlineDate != "" || app.dramas[1].Views != "0" || app.dramas[1].SortMetadata.Version != sortMetadataVersion {
|
||||
t.Fatal("resume changed history, lost zero, or invented an empty source date")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrimarySortFieldsNeedNoAdditionalInterfacesWhenComplete(t *testing.T) {
|
||||
hongguo := hongguoDramaFromAny(map[string]any{"series_id": "7615465407347952664", "series_title": "列表剧", "series_cover": coverAddressFixture, "first_visible_time": "1773662280", "hot_score": "20019", "series_play_cnt": "0"}, "测试")
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
t.Errorf("complete primary data triggered an optional request: %s", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 404, ""), nil
|
||||
})
|
||||
for _, drama := range []Drama{hongguo} {
|
||||
if drama.OnlineDate != "2026-03-16" || drama.Views != "0" || !hasCompleteSortMetadata(drama) {
|
||||
t.Errorf("primary sort fields were lost: %+v", drama)
|
||||
}
|
||||
if _, err := d.fetchDramaSortMetadata(context.Background(), drama); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if hasSortMetric("未知") || hasSortMetric("-1") {
|
||||
t.Fatal("unknown metrics were misclassified as available sorting data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackgroundRequestsYieldToForeground(t *testing.T) {
|
||||
limiter := newRequestLimiter(1, time.Nanosecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
request := httptest.NewRequest("GET", "https://hongguoduanju.com/search/test", nil)
|
||||
release, err := limiter.acquire(ctx, request)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
background := make(chan func(), 1)
|
||||
foreground := make(chan func(), 1)
|
||||
go func() {
|
||||
unlock, _ := limiter.acquire(context.WithValue(ctx, backgroundCatalogKey{}, true), request)
|
||||
background <- unlock
|
||||
}()
|
||||
go func() {
|
||||
unlock, _ := limiter.acquire(ctx, request)
|
||||
foreground <- unlock
|
||||
}()
|
||||
|
||||
for {
|
||||
limiter.mu.Lock()
|
||||
waiting := limiter.foregroundWaiting
|
||||
limiter.mu.Unlock()
|
||||
if waiting > 0 {
|
||||
break
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
t.Fatal("foreground waiter did not register")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case unlock := <-foreground:
|
||||
if unlock == nil {
|
||||
t.Fatal("foreground timed out")
|
||||
}
|
||||
select {
|
||||
case <-background:
|
||||
t.Fatal("background ran alongside a waiting foreground request")
|
||||
default:
|
||||
}
|
||||
unlock()
|
||||
case <-background:
|
||||
t.Fatal("optional metadata overtook foreground work")
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
select {
|
||||
case unlock := <-background:
|
||||
if unlock == nil {
|
||||
t.Fatal("background did not resume")
|
||||
}
|
||||
unlock()
|
||||
case <-ctx.Done():
|
||||
t.Fatal(ctx.Err())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptionalDetailPermissionFailureDoesNotBlockForeground(t *testing.T) {
|
||||
limiter := newRequestLimiter(1, time.Nanosecond)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
foreground := httptest.NewRequest("GET", "https://hongguoduanju.com/search/test", nil).WithContext(ctx)
|
||||
background := foreground.WithContext(context.WithValue(ctx, backgroundCatalogKey{}, true))
|
||||
limiter.observe(background, &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}})
|
||||
if release, err := limiter.acquire(background.Context(), background); err == nil {
|
||||
release()
|
||||
t.Fatal("repeated forbidden details must cool down")
|
||||
}
|
||||
release, err := limiter.acquire(ctx, foreground)
|
||||
if err != nil {
|
||||
t.Fatal("an optional detail's permissions blocked public search", err)
|
||||
}
|
||||
release()
|
||||
limiter.observe(background, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": {"60"}}})
|
||||
if release, err := limiter.acquire(ctx, foreground); err == nil {
|
||||
release()
|
||||
t.Fatal("site rate limits must still be respected")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func sortDetailFixture(id, stamp string) string {
|
||||
return `<script>window._ROUTER_DATA={"loaderData":{"detail_page":{"seriesDetail":{"series_id":"` + id + `","series_name":"测试剧","first_visible_time":"` + stamp + `","create_time":"1745906791"}}}}</script>`
|
||||
}
|
||||
|
||||
func TestSortDetailDatesAndIdentity(t *testing.T) {
|
||||
id := "7615465407347952664"
|
||||
drama, err := parseHongguoSortDetail(sortDetailFixture(id, "1773662280"), id)
|
||||
if err != nil || drama.OnlineDate != "2026-03-16" {
|
||||
t.Fatalf("first-visible date: %q, %v", drama.OnlineDate, err)
|
||||
}
|
||||
if _, err = parseHongguoSortDetail(sortDetailFixture("7498612570866076734", "1773662280"), id); err == nil {
|
||||
t.Fatal("accepted another drama's time")
|
||||
}
|
||||
drama, err = parseHongguoSortDetail(sortDetailFixture(id, "0"), id)
|
||||
if err != nil || drama.OnlineDate != "" {
|
||||
t.Fatal("creation time must not replace missing first-visible time")
|
||||
}
|
||||
for _, value := range []string{"", "0", "0001-01-01", "1970-01-01", "2101-01-01", "2026-02-31", "not a date"} {
|
||||
if got := providerReleaseDate(value); got != "" {
|
||||
t.Errorf("accepted date %q", value)
|
||||
}
|
||||
}
|
||||
if normalizeViews("10019") != "10019次播放" {
|
||||
t.Fatal("play counts must retain precision")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSortMetadataSelectionAndResume(t *testing.T) {
|
||||
now := time.Now()
|
||||
var rows []Drama
|
||||
for index := 0; index < 60; index++ {
|
||||
rows = append(rows, Drama{ID: fmt.Sprintf("hongguo:%019d", index+1), Source: sourceHongguo})
|
||||
}
|
||||
rows[1].SortMetadata = &sortMetadataState{Version: sortMetadataVersion, CheckedAt: now, CoverChecked: true}
|
||||
rows[2].SortMetadata = &sortMetadataState{CheckedAt: now}
|
||||
rows = append(rows, Drama{ID: "huangdou:other", Source: "huangdou"})
|
||||
batch := selectSortMetadataBatch(rows, sourceHongguo, []string{rows[59].ID, rows[59].ID, rows[2].ID, "huangdou:other"}, now)
|
||||
if len(batch) != sortMetadataBatchSize || batch[0].ID != rows[59].ID {
|
||||
t.Fatal("did not prioritize filtered entries within the batch limit")
|
||||
}
|
||||
for _, row := range batch {
|
||||
if row.ID == rows[1].ID || row.ID == rows[2].ID || row.Source != sourceHongguo {
|
||||
t.Fatal("selected completed, cooling-down or other-source entries")
|
||||
}
|
||||
}
|
||||
if got := selectSortMetadataBatch(rows, "huangdou", nil, now); len(got) != 0 {
|
||||
t.Fatal("source filter was not respected")
|
||||
}
|
||||
if remaining := sortMetadataRemaining(rows); remaining["hongguo"] != 59 || remaining["huangdou"] != 0 || remaining[""] != 59 {
|
||||
t.Fatal(remaining)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualBackfillPersistsPartialResultsAndContinuesAfterCatalogEnd(t *testing.T) {
|
||||
first, bad, unknown := "7615465407347952664", "7498612570866076734", "7677914379060251673"
|
||||
var calls atomic.Int32
|
||||
var fail atomic.Bool
|
||||
fail.Store(true)
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
calls.Add(1)
|
||||
switch request.URL.Path {
|
||||
case "/detail":
|
||||
id := request.URL.Query().Get("series_id")
|
||||
stamp := "1773662280"
|
||||
if id == unknown {
|
||||
stamp = "0"
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, sortDetailFixture(id, stamp)), nil
|
||||
case "/novel/player/video_detail/v1/":
|
||||
var body struct {
|
||||
ID string `json:"series_id"`
|
||||
}
|
||||
json.NewDecoder(request.Body).Decode(&body)
|
||||
if body.ID == bad && fail.Load() {
|
||||
return rankingHTTPResponse(request, 503, "temporary failure"), nil
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":%q,"series_title":"测试剧","hot_score":20019,"series_play_cnt":10019}}}`, body.ID)), nil
|
||||
default:
|
||||
t.Errorf("backfill requested a non-detail resource: %s", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 404, ""), nil
|
||||
}
|
||||
})
|
||||
client := d.hongguoClient()
|
||||
client.mu.Lock()
|
||||
for _, genre := range hongguoAppGenres {
|
||||
client.state.Feeds[genre.key] = hongguoCatalogCursor{Initialized: true, Exhausted: true}
|
||||
}
|
||||
client.mu.Unlock()
|
||||
a := &UIApp{downloader: d, cfg: d.cfg, libraryAttempted: true, librarySources: map[string]librarySourceState{}, libraryApp: d.hongguoCatalogSnapshot(), dramas: []Drama{
|
||||
{ID: "hongguo:" + first, Source: sourceHongguo, Title: "原有标题", Views: "0"},
|
||||
{ID: "hongguo:" + bad, Source: sourceHongguo, Title: "暂时失败", Heat: "800"},
|
||||
{ID: "hongguo:" + unknown, Source: sourceHongguo, Title: "站点日期未知"},
|
||||
}}
|
||||
t.Cleanup(a.stopSortMetadata)
|
||||
plain := httptest.NewRecorder()
|
||||
a.handleDramas(plain, httptest.NewRequest("GET", "/api/ui/dramas", nil))
|
||||
if calls.Load() != 0 {
|
||||
t.Fatal("ordinary library reads must not backfill")
|
||||
}
|
||||
before := a.librarySnapshotLocked(0)
|
||||
if before["hasMoreBySource"].(map[string]bool)[sourceHongguo] != true {
|
||||
t.Fatal("backfill must remain available after the catalog is exhausted")
|
||||
}
|
||||
writer := httptest.NewRecorder()
|
||||
a.handleDramas(writer, httptest.NewRequest("GET", "/api/ui/dramas?more=1&source=hongguo&priority=hongguo:"+first, nil))
|
||||
if writer.Code != 200 {
|
||||
t.Fatal(writer.Body.String())
|
||||
}
|
||||
waitSortMetadataQueue(t, a)
|
||||
a.mu.Lock()
|
||||
rows := append([]Drama(nil), a.dramas...)
|
||||
progress := a.libraryMetadata
|
||||
a.mu.Unlock()
|
||||
if progress.Checked != 3 || progress.Failed != 1 {
|
||||
t.Fatalf("unexpected progress: %+v", progress)
|
||||
}
|
||||
for _, row := range rows {
|
||||
switch row.ID {
|
||||
case "hongguo:" + first:
|
||||
if row.OnlineDate != "2026-03-16" || row.Heat != "20019" || row.Views != "10019次播放" || row.SortMetadata.Version != sortMetadataVersion {
|
||||
t.Fatal("fresh sorting fields were not merged")
|
||||
}
|
||||
case "hongguo:" + bad:
|
||||
if row.OnlineDate == "" || row.Heat != "800" || row.SortMetadata.Version != 0 {
|
||||
t.Fatal("partial success must survive a failed metrics request")
|
||||
}
|
||||
case "hongguo:" + unknown:
|
||||
if row.OnlineDate != "" || row.SortMetadata.Version != sortMetadataVersion {
|
||||
t.Fatal("a valid response with no date must not be retried forever or invent a date")
|
||||
}
|
||||
}
|
||||
}
|
||||
saved, err := readLibraryCache(d.cfg.dataDirectory())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if batch := selectSortMetadataBatch(saved.Dramas, sourceHongguo, nil, time.Now()); len(batch) != 0 {
|
||||
t.Fatal("restart lost completed checks or failure cooldown")
|
||||
}
|
||||
for index, row := range saved.Dramas {
|
||||
if row.ID == "hongguo:"+bad {
|
||||
saved.Dramas[index].SortMetadata = &sortMetadataState{CheckedAt: time.Now().Add(-sortMetadataRetryDelay - time.Second)}
|
||||
}
|
||||
}
|
||||
fail.Store(false)
|
||||
previous := calls.Load()
|
||||
patches, failures := d.backfillSortMetadata(withKnownHongguoDramas(context.Background(), saved.Dramas), sourceHongguo)
|
||||
if len(failures) != 0 || len(patches) != 1 || patches[0].ID != "hongguo:"+bad || calls.Load()-previous != 1 {
|
||||
t.Fatal("retry did not resume only the failed entry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualMetadataPriorityValidation(t *testing.T) {
|
||||
a := &UIApp{}
|
||||
for _, path := range []string{
|
||||
"/api/ui/dramas?more=1&source=invalid",
|
||||
"/api/ui/dramas?more=1&refresh=1",
|
||||
"/api/ui/dramas?update=1&refresh=1",
|
||||
"/api/ui/dramas?update=1&more=1",
|
||||
"/api/ui/dramas?update=1&source=invalid",
|
||||
"/api/ui/dramas?update=1&priority=" + strings.Repeat("hongguo:700001,", sortMetadataBatchSize),
|
||||
"/api/ui/dramas?update=1&priority=hongguo:" + strings.Repeat("1", 121),
|
||||
"/api/ui/dramas?more=1&priority=" + strings.Repeat("id,", sortMetadataBatchSize),
|
||||
} {
|
||||
writer := httptest.NewRecorder()
|
||||
a.handleDramas(writer, httptest.NewRequest("GET", path, nil))
|
||||
if writer.Code != 400 {
|
||||
t.Fatalf("invalid request was not rejected: %d", writer.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type librarySourceState struct {
|
||||
Status string `json:"status"`
|
||||
Count int `json:"count"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Phase string `json:"phase,omitempty"`
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
func (state librarySourceState) withPhase(phase string) librarySourceState {
|
||||
phase = strings.TrimSpace(phase)
|
||||
if state.Status == "loading" {
|
||||
state.Phase = phase
|
||||
}
|
||||
return state
|
||||
}
|
||||
|
||||
type libraryProgressKey struct{}
|
||||
type libraryCategoryKey struct{}
|
||||
type libraryProgressFunc func(string, []Drama, error, bool) error
|
||||
|
||||
type libraryLoadMode int
|
||||
|
||||
const (
|
||||
libraryLoadRefresh libraryLoadMode = iota
|
||||
libraryLoadMore
|
||||
libraryLoadUpdate
|
||||
)
|
||||
|
||||
func reportLibraryProgress(ctx context.Context, source string, items []Drama, err error, done bool) error {
|
||||
if callback, ok := ctx.Value(libraryProgressKey{}).(libraryProgressFunc); ok && (done || len(items) > 0) {
|
||||
return callback(source, items, err, done)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *UIApp) setLibraryPhaseLocked(phase string) {
|
||||
phase = strings.TrimSpace(phase)
|
||||
a.libraryLoadingPhase = phase
|
||||
if a.librarySources == nil {
|
||||
a.librarySources = map[string]librarySourceState{}
|
||||
}
|
||||
for source, state := range a.librarySources {
|
||||
if state.Status == "loading" {
|
||||
a.librarySources[source] = state.withPhase(phase)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *UIApp) handleLibraryCancel(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"})
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
cancel := a.libraryCancel
|
||||
running := a.libraryLoading != nil
|
||||
if running {
|
||||
a.setLibraryPhaseLocked("正在停止,已保留已入库内容")
|
||||
a.libraryRevision++
|
||||
}
|
||||
resp := a.librarySnapshotForSourceLocked(r.Context(), 0)
|
||||
a.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
resp["canceled"] = running
|
||||
writeJSON(w, http.StatusOK, resp)
|
||||
}
|
||||
|
||||
func (a *UIApp) startLibraryRefreshLocked(source string) {
|
||||
a.startLibraryLoadLocked(source, libraryLoadRefresh, nil, "")
|
||||
}
|
||||
|
||||
func (a *UIApp) startLibraryLoadLocked(source string, mode libraryLoadMode, priority []string, category string, scopes ...context.Context) {
|
||||
category = strings.TrimSpace(category)
|
||||
base := context.Background()
|
||||
if len(scopes) > 0 && sourceScope(scopes[0]) != nil {
|
||||
base = context.WithValue(base, accountSourceContextKey{}, sourceScope(scopes[0]))
|
||||
}
|
||||
if mode == libraryLoadUpdate || mode == libraryLoadMore {
|
||||
if source == "" && sourceScopeRestricted(base) {
|
||||
for _, allowed := range sourceScope(base).Sources {
|
||||
a.enqueueHistoricalMetadataLocked(allowed, priority)
|
||||
}
|
||||
} else {
|
||||
a.enqueueHistoricalMetadataLocked(source, priority)
|
||||
}
|
||||
}
|
||||
more := mode == libraryLoadMore
|
||||
if more {
|
||||
if !matchesSourceFilter(sourceHongguo, source) || !hongguoCatalogCategoryHasMore(a.libraryApp, category) {
|
||||
return
|
||||
}
|
||||
source = sourceHongguo
|
||||
}
|
||||
previousCount := len(a.dramas)
|
||||
a.libraryAttempted = true
|
||||
a.libraryLoading = make(chan struct{})
|
||||
a.libraryMore = more
|
||||
a.libraryLoadingSource = source
|
||||
a.libraryLoadingPhase = "等待开始"
|
||||
if a.librarySources == nil {
|
||||
a.librarySources = map[string]librarySourceState{}
|
||||
}
|
||||
for _, provider := range []string{sourceHongguo} {
|
||||
if matchesSourceFilter(provider, source) {
|
||||
state := a.librarySources[provider]
|
||||
state.Status = "loading"
|
||||
state.Error = ""
|
||||
state.Phase = "等待开始"
|
||||
a.librarySources[provider] = state
|
||||
}
|
||||
}
|
||||
a.libraryRevision++
|
||||
ctx, cancel := context.WithTimeout(base, 12*time.Minute)
|
||||
a.libraryCancel = cancel
|
||||
ctx = context.WithValue(ctx, libraryProgressKey{}, libraryProgressFunc(a.acceptLibraryProgress))
|
||||
ctx = context.WithValue(ctx, libraryMoreKey{}, more)
|
||||
ctx = context.WithValue(ctx, libraryUpdateKey{}, mode == libraryLoadUpdate)
|
||||
if category != "" {
|
||||
ctx = context.WithValue(ctx, libraryCategoryKey{}, category)
|
||||
}
|
||||
ctx = withKnownHongguoDramas(ctx, a.dramas)
|
||||
go func() {
|
||||
defer cancel()
|
||||
var loadErr error
|
||||
a.mu.Lock()
|
||||
a.setLibraryPhaseLocked("保存当前剧库")
|
||||
a.libraryRevision++
|
||||
a.mu.Unlock()
|
||||
if err := a.persistLibraryIfDirty(); err != nil {
|
||||
loadErr = err
|
||||
} else if more {
|
||||
a.mu.Lock()
|
||||
a.setLibraryPhaseLocked("继续加载目录")
|
||||
a.libraryRevision++
|
||||
a.mu.Unlock()
|
||||
items, err := a.downloader.fetchHongguoCategoryDramas(ctx, category)
|
||||
loadErr = errors.Join(err, reportLibraryProgress(ctx, sourceHongguo, items, err, true))
|
||||
} else {
|
||||
a.mu.Lock()
|
||||
a.setLibraryPhaseLocked("读取站源目录")
|
||||
a.libraryRevision++
|
||||
a.mu.Unlock()
|
||||
_, loadErr = a.downloader.fetchAllDramas(ctx, source)
|
||||
}
|
||||
a.mu.Lock()
|
||||
if ctx.Err() == context.Canceled {
|
||||
loadErr = nil
|
||||
a.markLibrarySourcesCanceledLocked(base, source)
|
||||
} else if loadErr != nil && a.libraryLoading != nil {
|
||||
a.markLibrarySourcesFailedLocked(base, source, loadErr)
|
||||
}
|
||||
a.libraryDirty = true
|
||||
a.libraryRevision++
|
||||
a.mu.Unlock()
|
||||
_ = a.persistLibrary()
|
||||
a.mu.Lock()
|
||||
close(a.libraryLoading)
|
||||
a.libraryLoading = nil
|
||||
a.libraryCancel = nil
|
||||
a.libraryLoadingPhase = ""
|
||||
a.libraryRevision++
|
||||
count := len(a.dramas)
|
||||
a.mu.Unlock()
|
||||
fmt.Printf("本地剧库:%d 部,本次新增 %d 部\n", count, count-previousCount)
|
||||
}()
|
||||
}
|
||||
|
||||
func (a *UIApp) persistLibraryIfDirty() error {
|
||||
a.mu.Lock()
|
||||
dirty := a.libraryDirty
|
||||
a.mu.Unlock()
|
||||
if !dirty {
|
||||
return nil
|
||||
}
|
||||
return a.persistLibrary()
|
||||
}
|
||||
|
||||
func (a *UIApp) markLibrarySourcesFailedLocked(ctx context.Context, source string, err error) {
|
||||
if a.librarySources == nil {
|
||||
a.librarySources = map[string]librarySourceState{}
|
||||
}
|
||||
if matchesSourceFilter(sourceHongguo, source) {
|
||||
state := a.librarySources[sourceHongguo]
|
||||
if state.Status == "loading" {
|
||||
state.Status = "failed"
|
||||
if state.Count > 0 {
|
||||
state.Status = "partial"
|
||||
}
|
||||
state.Error = a.redactError(err)
|
||||
state.Phase = ""
|
||||
a.librarySources[sourceHongguo] = state
|
||||
}
|
||||
}
|
||||
a.libraryError = librarySourceErrors(a.librarySources)
|
||||
}
|
||||
|
||||
func (a *UIApp) markLibrarySourcesCanceledLocked(ctx context.Context, source string) {
|
||||
if a.librarySources == nil {
|
||||
a.librarySources = map[string]librarySourceState{}
|
||||
}
|
||||
for _, provider := range []string{sourceHongguo} {
|
||||
if !matchesSourceFilter(provider, source) || !sourceAllowed(ctx, provider) {
|
||||
continue
|
||||
}
|
||||
state := a.librarySources[provider]
|
||||
if state.Status == "loading" {
|
||||
state.Status = "canceled"
|
||||
state.Error = ""
|
||||
state.Phase = ""
|
||||
a.librarySources[provider] = state
|
||||
}
|
||||
}
|
||||
a.libraryError = librarySourceErrors(a.librarySources)
|
||||
}
|
||||
|
||||
func (a *UIApp) acceptLibraryProgress(source string, items []Drama, loadErr error, done bool) error {
|
||||
if canonicalProviderSource(source) != sourceHongguo {
|
||||
return nil
|
||||
}
|
||||
source = sourceHongguo
|
||||
a.mu.Lock()
|
||||
items = onlyHongguoDramas(items)
|
||||
a.normalizeDramaCovers(items)
|
||||
newDramas := a.newSortMetadataDramasLocked(items)
|
||||
if source == sourceHongguo {
|
||||
a.libraryApp = a.downloader.hongguoCatalogSnapshot()
|
||||
}
|
||||
if done {
|
||||
var mergeErr error
|
||||
if loadErr != nil {
|
||||
mergeErr = &libraryLoadError{failures: map[string]error{source: loadErr}}
|
||||
}
|
||||
a.dramas = mergeSourceDramas(a.dramas, items, mergeErr, source)
|
||||
} else {
|
||||
positions := make(map[string]int, len(a.dramas))
|
||||
for index, drama := range a.dramas {
|
||||
positions[drama.ID] = index
|
||||
}
|
||||
for _, drama := range items {
|
||||
if drama.ID == "" {
|
||||
continue
|
||||
}
|
||||
if position, found := positions[drama.ID]; found {
|
||||
a.dramas[position] = mergeDramaMetadata(drama, a.dramas[position])
|
||||
} else {
|
||||
positions[drama.ID] = len(a.dramas)
|
||||
a.dramas = append(a.dramas, drama)
|
||||
}
|
||||
}
|
||||
}
|
||||
a.enqueueSortMetadataLocked(newDramas, false)
|
||||
state := a.librarySources[source]
|
||||
state.Count = 0
|
||||
for _, drama := range a.dramas {
|
||||
if dramaProvider(drama) == source {
|
||||
state.Count++
|
||||
}
|
||||
}
|
||||
if len(items) > 0 {
|
||||
a.loadedAt = time.Now()
|
||||
state.UpdatedAt = a.loadedAt
|
||||
}
|
||||
if done {
|
||||
state.Status = "ready"
|
||||
if loadErr != nil {
|
||||
state.Status = "failed"
|
||||
if state.Count > 0 {
|
||||
state.Status = "partial"
|
||||
}
|
||||
state.Error = a.redactError(loadErr)
|
||||
}
|
||||
state.Phase = ""
|
||||
}
|
||||
a.librarySources[source] = state
|
||||
a.libraryError = librarySourceErrors(a.librarySources)
|
||||
a.libraryDirty = true
|
||||
a.libraryRevision++
|
||||
persist := done || len(items) > 0 && source == sourceHongguo || time.Since(a.libraryLastSave) >= 3*time.Second
|
||||
if persist {
|
||||
a.libraryLastSave = time.Now()
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if persist {
|
||||
return a.persistLibrary()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func librarySourceErrors(sources map[string]librarySourceState) string {
|
||||
var messages []string
|
||||
for source, state := range sources {
|
||||
if state.Error != "" {
|
||||
messages = append(messages, fmt.Sprintf("%s 获取失败: %s", source, state.Error))
|
||||
}
|
||||
}
|
||||
sort.Strings(messages)
|
||||
return strings.Join(messages, "\n")
|
||||
}
|
||||
|
||||
func cloneLibrarySources(sources map[string]librarySourceState) map[string]librarySourceState {
|
||||
copy := make(map[string]librarySourceState, len(sources))
|
||||
for source, state := range sources {
|
||||
copy[source] = state
|
||||
}
|
||||
return copy
|
||||
}
|
||||
|
||||
func (a *UIApp) persistLibrary() error {
|
||||
a.libraryWriteMu.Lock()
|
||||
defer a.libraryWriteMu.Unlock()
|
||||
a.mu.Lock()
|
||||
cache := libraryCache{Dramas: append([]Drama{}, a.dramas...), LoadedAt: a.loadedAt, LastError: librarySourceErrors(a.librarySources), Sources: cloneLibrarySources(a.librarySources), HongguoApp: cloneHongguoCatalogState(a.libraryApp)}
|
||||
revision := a.libraryRevision
|
||||
a.mu.Unlock()
|
||||
err := writeLibraryCache(a.cfg.dataDirectory(), cache)
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if err != nil {
|
||||
a.libraryError = strings.TrimSpace(cache.LastError + "\n剧库缓存保存失败: " + a.redactError(err))
|
||||
a.libraryRevision++
|
||||
return err
|
||||
}
|
||||
a.librarySaved = true
|
||||
if revision == a.libraryRevision {
|
||||
a.libraryDirty = false
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *UIApp) librarySnapshotLocked(revision uint64) map[string]any {
|
||||
remaining := sortMetadataRemaining(a.dramas)
|
||||
var unqueued []Drama
|
||||
for _, drama := range a.dramas {
|
||||
if !a.metadataPending[drama.ID] {
|
||||
unqueued = append(unqueued, drama)
|
||||
}
|
||||
}
|
||||
more := map[string]bool{}
|
||||
for source, count := range sortMetadataRemaining(unqueued) {
|
||||
more[source] = count > 0
|
||||
}
|
||||
more["hongguo"] = more["hongguo"] || hongguoCatalogHasMore(a.libraryApp)
|
||||
more[""] = more[""] || more["hongguo"]
|
||||
response := map[string]any{
|
||||
"revision": a.libraryRevision, "loadedAt": a.loadedAt, "loading": a.libraryLoading != nil,
|
||||
"cached": a.librarySaved && !a.libraryDirty, "error": a.libraryError,
|
||||
"sources": cloneLibrarySources(a.librarySources), "total": len(a.dramas),
|
||||
"hasMore": more["hongguo"], "hasMoreBySource": more, "metadataRemaining": remaining,
|
||||
"loadingMore": a.libraryMore && a.libraryLoading != nil, "loadingSource": a.libraryLoadingSource, "loadingPhase": a.libraryLoadingPhase, "metadata": a.libraryMetadata,
|
||||
}
|
||||
if revision != 0 && revision == a.libraryRevision {
|
||||
response["unchanged"] = true
|
||||
} else {
|
||||
response["data"] = append([]Drama{}, a.dramas...)
|
||||
}
|
||||
return response
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestUnifiedUpdateChecksNewContentAndContinuesCatalog(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
pages int
|
||||
savedOffset int
|
||||
savedEnd bool
|
||||
knownOffsets []int
|
||||
endAt int
|
||||
wantOffsets []int
|
||||
wantSaved int
|
||||
wantEnd bool
|
||||
resumeSession bool
|
||||
}{
|
||||
{name: "first import runs one batch", pages: 2, endAt: -1, wantOffsets: []int{0, 18}, wantSaved: 36},
|
||||
{name: "known head resumes saved tail", pages: 2, savedOffset: 90, knownOffsets: []int{0}, endAt: -1, wantOffsets: []int{0, 90, 108}, wantSaved: 126, resumeSession: true},
|
||||
{name: "new head stops at overlap", pages: 4, savedOffset: 90, knownOffsets: []int{18}, endAt: -1, wantOffsets: []int{0, 18, 90, 108, 126, 144}, wantSaved: 162, resumeSession: true},
|
||||
{name: "large update continues next unread page", pages: 4, savedOffset: 180, endAt: -1, wantOffsets: []int{0, 18, 36, 54, 72, 90, 108}, wantSaved: 126},
|
||||
{name: "exhausted catalog still checks new content", pages: 4, savedOffset: 180, savedEnd: true, knownOffsets: []int{18}, endAt: -1, wantOffsets: []int{0, 18}, wantSaved: 180, wantEnd: true},
|
||||
{name: "continuation reaches end", pages: 2, savedOffset: 90, knownOffsets: []int{0}, endAt: 90, wantOffsets: []int{0, 90}, wantSaved: 108, wantEnd: true, resumeSession: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
calls := map[string][]int{}
|
||||
var known []Drama
|
||||
for index := range hongguoAppGenres {
|
||||
for _, offset := range test.knownOffsets {
|
||||
known = append(known, Drama{ID: fmt.Sprintf("hongguo:%d", 790000+index*1000+offset), Source: sourceHongguo})
|
||||
}
|
||||
}
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
var payload struct {
|
||||
Offset int `json:"offset"`
|
||||
Session string `json:"session_id"`
|
||||
Select struct {
|
||||
Genre []string `json:"genre"`
|
||||
} `json:"select_items"`
|
||||
}
|
||||
if request.URL.Path != "/reading/distribution/category/landpage/v/" || json.NewDecoder(request.Body).Decode(&payload) != nil || len(payload.Select.Genre) != 1 {
|
||||
t.Errorf("unexpected catalog request: %s", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 400, ""), nil
|
||||
}
|
||||
genre := payload.Select.Genre[0]
|
||||
index := -1
|
||||
for position, candidate := range hongguoAppGenres {
|
||||
if candidate.key == genre {
|
||||
index = position
|
||||
}
|
||||
}
|
||||
if index < 0 {
|
||||
t.Errorf("unexpected genre: %s", genre)
|
||||
return rankingHTTPResponse(request, 400, ""), nil
|
||||
}
|
||||
calls[genre] = append(calls[genre], payload.Offset)
|
||||
if payload.Offset == 0 && payload.Session != "" {
|
||||
t.Error("head check reused a tail session")
|
||||
}
|
||||
if test.resumeSession && payload.Offset == test.savedOffset && payload.Session != "saved-"+genre {
|
||||
t.Error("continuation lost its saved session")
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"data": map[string]any{
|
||||
"next_offset": payload.Offset + 18, "session_id": "fixture-" + genre, "has_more": payload.Offset != test.endAt,
|
||||
"video_data": []any{map[string]any{
|
||||
"series_id": fmt.Sprint(790000 + index*1000 + payload.Offset), "series_title": "文字目录样本",
|
||||
"first_visible_time": "1773662280", "hot_score": "20", "series_play_cnt": "0",
|
||||
}},
|
||||
}})
|
||||
return rankingHTTPResponse(request, 200, string(body)), nil
|
||||
})
|
||||
d.cfg.MaxPagesPerSort = test.pages
|
||||
if test.savedOffset > 0 {
|
||||
client := d.hongguoClient()
|
||||
for _, genre := range hongguoAppGenres {
|
||||
client.state.Feeds[genre.key] = hongguoCatalogCursor{
|
||||
Offset: test.savedOffset, SessionID: "saved-" + genre.key, Initialized: true, Exhausted: test.savedEnd, UpdatedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
}
|
||||
ctx := withKnownHongguoDramas(context.WithValue(context.Background(), libraryUpdateKey{}, true), known)
|
||||
items, err := d.fetchHongguoAppCatalog(ctx)
|
||||
if err != nil || len(items) != len(test.wantOffsets)*len(hongguoAppGenres) {
|
||||
t.Fatalf("catalog result: %d items, %v", len(items), err)
|
||||
}
|
||||
for _, genre := range hongguoAppGenres {
|
||||
if !reflect.DeepEqual(calls[genre.key], test.wantOffsets) {
|
||||
t.Errorf("%s requested %v, want %v", genre.key, calls[genre.key], test.wantOffsets)
|
||||
}
|
||||
cursor := d.hongguoCatalogSnapshot().Feeds[genre.key]
|
||||
if !cursor.Initialized || cursor.Offset != test.wantSaved || cursor.Exhausted != test.wantEnd {
|
||||
t.Errorf("wrong checkpoint for %s: %+v", genre.key, cursor)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedUpdatePreservesHistoryAcrossRestart(t *testing.T) {
|
||||
calls := map[string][]int{}
|
||||
transport := func(request *http.Request) (*http.Response, error) {
|
||||
var payload struct {
|
||||
Offset int `json:"offset"`
|
||||
Scene string `json:"req_scene"`
|
||||
}
|
||||
if request.URL.Path != "/reading/distribution/category/landpage/v/" || json.NewDecoder(request.Body).Decode(&payload) != nil {
|
||||
t.Errorf("unexpected request: %s", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 400, ""), nil
|
||||
}
|
||||
calls[payload.Scene] = append(calls[payload.Scene], payload.Offset)
|
||||
index := 0
|
||||
for position, genre := range hongguoAppGenres {
|
||||
if genre.scene == payload.Scene {
|
||||
index = position
|
||||
}
|
||||
}
|
||||
body := fmt.Sprintf(`{"data":{"next_offset":%d,"has_more":%t,"session_id":"saved-session","video_data":[{"series_id":"%d","series_title":"文字样本","first_visible_time":"1773662280","hot_score":"20","series_play_cnt":"0"}]}}`, payload.Offset+18, payload.Offset == 0, 790000+index*1000+payload.Offset)
|
||||
return rankingHTTPResponse(request, 200, body), nil
|
||||
}
|
||||
directory := t.TempDir()
|
||||
for run, wantCount := range []int{3, 6, 6} {
|
||||
d := rankingTestDownloader(t, transport)
|
||||
d.cfg.dataDir, d.cfg.MaxPagesPerSort = directory, 1
|
||||
items, err := d.RefreshDramas(context.WithValue(context.Background(), libraryUpdateKey{}, true), sourceHongguo)
|
||||
if err != nil || len(items) != wantCount {
|
||||
t.Fatalf("run %d lost history: %d items, %v", run, len(items), err)
|
||||
}
|
||||
cache, err := readLibraryCache(directory)
|
||||
if err != nil || len(cache.Dramas) != wantCount || hongguoCatalogHasMore(cache.HongguoApp) != (run == 0) {
|
||||
t.Fatalf("run %d did not persist data and cursor together: %v", run, err)
|
||||
}
|
||||
for _, drama := range cache.Dramas {
|
||||
if drama.OnlineDate != "2026-03-16" || drama.Heat != "20" || drama.Views != "0" {
|
||||
t.Fatal("update lost available sorting fields")
|
||||
}
|
||||
}
|
||||
}
|
||||
for scene, offsets := range calls {
|
||||
if !reflect.DeepEqual(offsets, []int{0, 0, 18, 0}) {
|
||||
t.Errorf("%s skipped or repeated a continuation: %v", scene, offsets)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnifiedUpdateReturnsImmediatelyAndDeduplicatesWork(t *testing.T) {
|
||||
catalogStarted, catalogRelease := make(chan struct{}), make(chan struct{})
|
||||
detailStarted, detailRelease := make(chan string, 1), make(chan struct{})
|
||||
releaseCatalog := sync.OnceFunc(func() { close(catalogRelease) })
|
||||
releaseDetail := sync.OnceFunc(func() { close(detailRelease) })
|
||||
var catalogOnce sync.Once
|
||||
var catalogCalls atomic.Int32
|
||||
var detailMu sync.Mutex
|
||||
var detailIDs []string
|
||||
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
|
||||
switch request.URL.Path {
|
||||
case "/reading/distribution/category/landpage/v/":
|
||||
catalogCalls.Add(1)
|
||||
catalogOnce.Do(func() { close(catalogStarted) })
|
||||
select {
|
||||
case <-catalogRelease:
|
||||
case <-request.Context().Done():
|
||||
return nil, request.Context().Err()
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, `{"data":{"next_offset":18,"has_more":false,"video_data":[{"series_id":"790000","series_title":"已完成的目录条目","first_visible_time":"1773662280","hot_score":"20","series_play_cnt":"0"}]}}`), nil
|
||||
case "/detail":
|
||||
if request.Context().Value(backgroundCatalogKey{}) != true {
|
||||
t.Error("historical metadata did not use background priority")
|
||||
}
|
||||
id := request.URL.Query().Get("series_id")
|
||||
detailMu.Lock()
|
||||
detailIDs = append(detailIDs, id)
|
||||
if len(detailIDs) == 1 {
|
||||
detailStarted <- id
|
||||
}
|
||||
detailMu.Unlock()
|
||||
select {
|
||||
case <-detailRelease:
|
||||
case <-request.Context().Done():
|
||||
return nil, request.Context().Err()
|
||||
}
|
||||
return rankingHTTPResponse(request, 200, sortDetailFixture(id, "1773662280")), nil
|
||||
default:
|
||||
t.Errorf("unexpected resource: %s", request.URL.Path)
|
||||
return rankingHTTPResponse(request, 404, ""), nil
|
||||
}
|
||||
})
|
||||
d.cfg.MaxPagesPerSort = 1
|
||||
for _, genre := range hongguoAppGenres {
|
||||
d.hongguoClient().state.Feeds[genre.key] = hongguoCatalogCursor{Initialized: true, Exhausted: true, Offset: 18}
|
||||
}
|
||||
app := &UIApp{downloader: d, cfg: d.cfg, libraryAttempted: true, libraryApp: d.hongguoCatalogSnapshot(), librarySources: map[string]librarySourceState{}}
|
||||
app.dramas = append(app.dramas, Drama{ID: "hongguo:790000", Source: sourceHongguo, Title: "已完成的目录条目", OnlineDate: "2026-03-16", Heat: "20", Views: "0", Cover: coverAddressFixture, SortMetadata: &sortMetadataState{Version: sortMetadataVersion}})
|
||||
for index := 0; index < 60; index++ {
|
||||
app.dramas = append(app.dramas, Drama{ID: fmt.Sprintf("hongguo:%d", 800000+index), Source: sourceHongguo, Title: "历史文字样本", Heat: "20", Views: "0", Cover: coverAddressFixture})
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
releaseCatalog()
|
||||
releaseDetail()
|
||||
app.mu.Lock()
|
||||
done, cancel := app.libraryLoading, app.libraryCancel
|
||||
app.mu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
if done != nil {
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Error("catalog worker did not stop")
|
||||
}
|
||||
}
|
||||
app.stopSortMetadata()
|
||||
})
|
||||
type snapshot struct {
|
||||
Accepted bool `json:"updateAccepted"`
|
||||
Loading bool `json:"loading"`
|
||||
Data []Drama `json:"data"`
|
||||
Metadata libraryMetadataProgress `json:"metadata"`
|
||||
}
|
||||
read := func(path string) snapshot {
|
||||
t.Helper()
|
||||
done := make(chan *httptest.ResponseRecorder, 1)
|
||||
go func() {
|
||||
writer := httptest.NewRecorder()
|
||||
app.handleDramas(writer, httptest.NewRequest("GET", path, nil))
|
||||
done <- writer
|
||||
}()
|
||||
select {
|
||||
case writer := <-done:
|
||||
var result snapshot
|
||||
if writer.Code != 200 || json.Unmarshal(writer.Body.Bytes(), &result) != nil {
|
||||
t.Fatalf("invalid snapshot: %d %s", writer.Code, writer.Body.String())
|
||||
}
|
||||
return result
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("library response waited for a stalled upstream request")
|
||||
return snapshot{}
|
||||
}
|
||||
}
|
||||
if result := read("/api/ui/dramas"); result.Loading || result.Metadata.Total != 0 || catalogCalls.Load() != 0 {
|
||||
t.Fatal("ordinary cache read scheduled remote work")
|
||||
}
|
||||
path := "/api/ui/dramas?update=1&source=hongguo&priority=hongguo:800059"
|
||||
result := read(path)
|
||||
if !result.Accepted || !result.Loading || result.Metadata.Total != 40 || len(result.Data) != 61 {
|
||||
t.Fatalf("update did not immediately return cache and bounded progress: %+v", result.Metadata)
|
||||
}
|
||||
select {
|
||||
case <-catalogStarted:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("update did not start the main catalog")
|
||||
}
|
||||
for _, request := range []string{path, path, "/api/ui/dramas?revision=1"} {
|
||||
if result := read(request); result.Accepted || !result.Loading || result.Metadata.Total != 40 {
|
||||
t.Fatal("repeated update or polling duplicated work")
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-detailStarted:
|
||||
t.Fatal("metadata did not yield to the catalog")
|
||||
default:
|
||||
}
|
||||
app.mu.Lock()
|
||||
catalogDone := app.libraryLoading
|
||||
app.mu.Unlock()
|
||||
releaseCatalog()
|
||||
select {
|
||||
case <-catalogDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("catalog waited for optional metadata")
|
||||
}
|
||||
select {
|
||||
case id := <-detailStarted:
|
||||
if id != "800059" {
|
||||
t.Fatalf("current filtered result was not prioritized: %s", id)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("historical metadata was not scheduled")
|
||||
}
|
||||
if result := read("/api/ui/dramas"); result.Loading || !result.Metadata.Running || result.Metadata.Checked != 0 || catalogCalls.Load() != 3 {
|
||||
t.Fatal("optional metadata blocked the main response or catalog work was duplicated")
|
||||
}
|
||||
releaseDetail()
|
||||
waitSortMetadataQueue(t, app)
|
||||
result = read("/api/ui/dramas")
|
||||
if result.Metadata.Checked != 40 || result.Metadata.Failed != 0 || result.Metadata.Running {
|
||||
t.Fatalf("unexpected completion: %+v", result.Metadata)
|
||||
}
|
||||
untouched := 0
|
||||
for _, drama := range result.Data {
|
||||
if strings.HasPrefix(drama.ID, "hongguo:800") && drama.SortMetadata == nil {
|
||||
untouched++
|
||||
}
|
||||
}
|
||||
if untouched != 20 {
|
||||
t.Fatalf("update migrated more than one historical batch: %d untouched", untouched)
|
||||
}
|
||||
detailMu.Lock()
|
||||
defer detailMu.Unlock()
|
||||
seen := map[string]bool{}
|
||||
for _, id := range detailIDs {
|
||||
if seen[id] {
|
||||
t.Fatalf("duplicate historical request: %s", id)
|
||||
}
|
||||
seen[id] = true
|
||||
}
|
||||
if len(seen) != 40 {
|
||||
t.Fatalf("unexpected detail request count: %d", len(seen))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"errors"
|
||||
)
|
||||
|
||||
func pkcs7Unpad(src []byte, blockSize int) ([]byte, error) {
|
||||
if len(src) == 0 || len(src)%blockSize != 0 {
|
||||
return nil, errors.New("invalid pkcs7 length")
|
||||
}
|
||||
pad := int(src[len(src)-1])
|
||||
if pad == 0 || pad > blockSize || pad > len(src) {
|
||||
return nil, errors.New("invalid pkcs7 padding")
|
||||
}
|
||||
for _, v := range src[len(src)-pad:] {
|
||||
if int(v) != pad {
|
||||
return nil, errors.New("invalid pkcs7 padding bytes")
|
||||
}
|
||||
}
|
||||
return src[:len(src)-pad], nil
|
||||
}
|
||||
Reference in New Issue
Block a user