diff --git a/internal/app/library_cache.go b/internal/app/library_cache.go new file mode 100644 index 0000000..189d933 --- /dev/null +++ b/internal/app/library_cache.go @@ -0,0 +1,170 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" +) + +type libraryCache struct { + Dramas []Drama `json:"dramas"` + LoadedAt time.Time `json:"loadedAt"` + LastError string `json:"lastError,omitempty"` + Sources map[string]librarySourceState `json:"sources,omitempty"` + HongguoApp *hongguoCatalogState `json:"hongguoApp,omitempty"` +} + +func libraryCachePath(outputDir string) string { + return filepath.Join(outputDir, "library.json") +} + +func readLibraryCache(outputDir string) (libraryCache, error) { + var cache libraryCache + path := libraryCachePath(outputDir) + body, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + path = filepath.Join(outputDir, "ui-state.json") + body, err = os.ReadFile(path) + } + if err != nil { + return cache, err + } + if err := json.Unmarshal(body, &cache); err != nil { + return cache, fmt.Errorf("本地剧库缓存损坏 %s: %w", path, err) + } + originalCount := len(cache.Dramas) + cache.Dramas = onlyHongguoDramas(cache.Dramas) + sources := make(map[string]librarySourceState) + for source, state := range cache.Sources { + if canonicalProviderSource(source) == sourceHongguo { + sources[sourceHongguo] = state + } + } + if originalCount != len(cache.Dramas) || len(sources) != len(cache.Sources) { + cache.LastError = librarySourceErrors(sources) + } + cache.Sources = sources + return cache, nil +} + +func writeLibraryCache(outputDir string, cache libraryCache) error { + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return err + } + file, err := os.CreateTemp(outputDir, ".library-*.tmp") + if err != nil { + return err + } + defer os.Remove(file.Name()) + encoder := json.NewEncoder(file) + if err := encoder.Encode(cache); err != nil { + _ = file.Close() + return err + } + if err := file.Sync(); err != nil { + _ = file.Close() + return err + } + if err := file.Close(); err != nil { + return err + } + return os.Rename(file.Name(), libraryCachePath(outputDir)) +} + +func (a *UIApp) loadLibrary() { + cache, err := readLibraryCache(a.cfg.dataDirectory()) + if err == nil { + a.downloader.restoreHongguoCatalog(cache.HongguoApp) + } + a.mu.Lock() + defer a.mu.Unlock() + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + a.libraryError = a.redactError(err) + } + return + } + if len(cache.Dramas) > 0 && (len(a.dramas) == 0 || !a.loadedAt.After(cache.LoadedAt)) { + a.dramas = cache.Dramas + a.normalizeDramaCovers(a.dramas) + a.loadedAt = cache.LoadedAt + a.libraryError = cache.LastError + } + a.librarySources = cache.Sources + if cache.HongguoApp != nil { + a.libraryApp = a.downloader.hongguoCatalogSnapshot() + } + for source, state := range a.librarySources { + if state.Status == "loading" { + state.Status = "partial" + state.Error = "上次更新中断,当前显示已保存数据,可手动刷新" + a.librarySources[source] = state + } + } + if sourceErrors := librarySourceErrors(a.librarySources); sourceErrors != "" { + a.libraryError = sourceErrors + } + a.libraryRevision = 1 + if len(a.dramas) > 0 { + if _, err := os.Stat(libraryCachePath(a.cfg.dataDirectory())); errors.Is(err, os.ErrNotExist) { + if err := writeLibraryCache(a.cfg.dataDirectory(), libraryCache{Dramas: a.dramas, LoadedAt: a.loadedAt, LastError: a.libraryError, Sources: a.librarySources, HongguoApp: a.libraryApp}); err != nil { + a.libraryError = "剧库缓存保存失败: " + a.redactError(err) + return + } + } + a.librarySaved = true + var pending []Drama + for _, drama := range a.dramas { + if drama.SortMetadata != nil && drama.SortMetadata.Pending { + pending = append(pending, drama) + } + } + a.enqueueSortMetadataLocked(pending, false) + fmt.Printf("已恢复本地剧库:%d 部,缓存:%s\n", len(a.dramas), libraryCachePath(a.cfg.dataDirectory())) + } +} + +func (d *Downloader) GetAllDramas(ctx context.Context) ([]Drama, error) { + cache, err := readLibraryCache(d.cfg.dataDirectory()) + if err == nil { + d.restoreHongguoCatalog(cache.HongguoApp) + } + if err == nil && len(cache.Dramas) > 0 { + fmt.Printf("使用本地剧库缓存:%d 部(%s),更新请使用 -refresh\n", len(cache.Dramas), cache.LoadedAt.Format("2006-01-02 15:04:05")) + if cache.LastError != "" { + return cache.Dramas, errors.New(cache.LastError) + } + return cache.Dramas, nil + } + if err != nil && !errors.Is(err, os.ErrNotExist) { + fmt.Printf(" 读取本地剧库缓存失败: %v\n", err) + } + return d.RefreshDramas(ctx, "") +} + +func (d *Downloader) RefreshDramas(ctx context.Context, source string) ([]Drama, error) { + cached, _ := readLibraryCache(d.cfg.dataDirectory()) + d.restoreHongguoCatalog(cached.HongguoApp) + ctx = withKnownHongguoDramas(ctx, cached.Dramas) + fresh, loadErr := d.fetchAllDramas(ctx, source) + merged := mergeSourceDramas(cached.Dramas, fresh, loadErr, source) + if len(fresh) == 0 && len(merged) == 0 { + return nil, loadErr + } + cache := libraryCache{Dramas: merged, LoadedAt: time.Now(), HongguoApp: d.hongguoCatalogSnapshot()} + if len(fresh) == 0 { + cache.LoadedAt = cached.LoadedAt + } + if loadErr != nil { + cache.LastError = publicError(loadErr).Error() + } + if err := writeLibraryCache(d.cfg.dataDirectory(), cache); err != nil { + loadErr = errors.Join(loadErr, fmt.Errorf("剧库缓存保存失败: %w", err)) + } + fmt.Printf("本地剧库:%d 部,本次新增 %d 部\n", len(merged), len(merged)-len(cached.Dramas)) + return merged, loadErr +} diff --git a/internal/app/library_cover_metadata_test.go b/internal/app/library_cover_metadata_test.go new file mode 100644 index 0000000..a44a3e4 --- /dev/null +++ b/internal/app/library_cover_metadata_test.go @@ -0,0 +1,109 @@ +package app + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync/atomic" + "testing" + "time" +) + +const coverAddressFixture = "https://p3-reading-sign.fqnovelpic.com/synthetic-cover-address?fixture=1" + +func TestHongguoRankingKeepsCoverAddressForLibrary(t *testing.T) { + board, _ := findRankingBoard("hongguo-hot") + rows := rankingRows(1) + rows[0]["cover"] = coverAddressFixture + page, err := parseHongguoRanking(rankingFixture(board, 1, rows), board, 1) + if err != nil || bestDramaCover(page.Items[0].Drama) != coverAddressFixture || bestDramaCover(page.Items[1].Drama) != "" { + t.Fatal("ranking discarded its cover address or invented one", err) + } + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + t.Error("retaining an address must not fetch the image", request.URL.Path) + return rankingHTTPResponse(request, 404, ""), nil + }) + app := &UIApp{downloader: d, cfg: d.cfg, metadataClosed: true} + page.BoardID, page.Page, page.FetchedAt = board.ID, 1, time.Now() + app.acceptRankingDramas(page) + for _, drama := range app.dramas { + if drama.ID == page.Items[0].Drama.ID && drama.Cover != "/api/ui/image?url="+url.QueryEscape(coverAddressFixture)+"&dramaId="+url.QueryEscape(drama.ID) { + t.Fatal("ranking address bypassed the local cover route") + } + } + for _, invalid := range []any{"javascript:invalid", "https://example.invalid/cover", "http://p3-reading-sign.fqnovelpic.com/cover", map[string]any{"unexpected": "field"}} { + rows[0]["cover"] = invalid + result, err := parseHongguoRanking(rankingFixture(board, 1, rows), board, 1) + if err != nil || len(result.Items) != 2 || bestDramaCover(result.Items[0].Drama) != "" { + t.Fatal("invalid optional address broke the ranking or reached the library", err) + } + } +} + +func TestHistoricalMissingCoverWaitsForManualUpdate(t *testing.T) { + for _, found := range []bool{true, false} { + t.Run(fmt.Sprintf("address=%t", found), func(t *testing.T) { + var calls atomic.Int32 + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.URL.Path != "/novel/player/video_detail/v1/" || request.Context().Value(backgroundCatalogKey{}) != true { + t.Error("cover metadata used a non-background or non-text request", request.URL.Path) + return rankingHTTPResponse(request, 404, ""), nil + } + cover := "" + if found { + cover = coverAddressFixture + } + return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":"7000000000000000001","series_title":"合成资料","series_cover":%q,"hot_score":999,"series_play_cnt":999}}}`, cover)), nil + }) + old := Drama{ID: "hongguo:7000000000000000001", Source: sourceHongguo, Title: "历史榜单条目", OnlineDate: "2026-09-01", Heat: "20019", Views: "0", + SortMetadata: &sortMetadataState{Version: sortMetadataVersion, CheckedAt: time.Now().Add(-time.Hour)}} + if err := writeLibraryCache(d.cfg.dataDirectory(), libraryCache{Dramas: []Drama{old}}); err != nil { + t.Fatal(err) + } + app := &UIApp{downloader: d, cfg: d.cfg} + t.Cleanup(app.stopSortMetadata) + app.loadLibrary() + app.handleDramas(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/ui/dramas", nil)) + app.mu.Lock() + if calls.Load() != 0 || app.metadataDone != nil || len(app.newSortMetadataDramasLocked([]Drama{old})) != 0 || sortMetadataRemaining(app.dramas)[sourceHongguo] != 1 { + app.mu.Unlock() + t.Fatal("ordinary loading migrated historical addresses or hid their pending update") + } + app.enqueueHistoricalMetadataLocked(sourceHongguo, []string{old.ID}) + app.mu.Unlock() + waitSortMetadataQueue(t, app) + cache, err := readLibraryCache(d.cfg.dataDirectory()) + if err != nil || len(cache.Dramas) != 1 { + t.Fatal("manual address repair was not persisted", err) + } + updated := cache.Dramas[0] + if calls.Load() != 1 || updated.SortMetadata == nil || !updated.SortMetadata.CoverChecked || needsSortMetadata(updated) || updated.Views != "0" || updated.Heat != old.Heat || updated.Title != old.Title { + t.Fatal("cover repair changed primary metadata or did not remember a completed check") + } + if found && (bestDramaCover(updated) != coverAddressFixture || !strings.HasPrefix(updated.Cover.(string), "/api/ui/image?url=")) || !found && bestDramaCover(updated) != "" { + t.Fatal("manual repair lost the address or invented a missing cover") + } + app.mu.Lock() + app.enqueueHistoricalMetadataLocked(sourceHongguo, []string{old.ID}) + app.mu.Unlock() + waitSortMetadataQueue(t, app) + if calls.Load() != 1 { + t.Fatal("known missing cover triggered repeated requests") + } + }) + } +} + +func TestHongguoCoverRepairValidatesDramaIdentity(t *testing.T) { + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":"7000000000000000002","series_cover":%q}}}`, coverAddressFixture)), nil + }) + patch, err := d.fetchDramaSortMetadata(context.Background(), Drama{ID: "hongguo:7000000000000000001", OnlineDate: "2026-09-01", Heat: "1", Views: "0"}) + if err == nil || bestDramaCover(patch) != "" { + t.Fatal("cover repair accepted another drama's address") + } +} diff --git a/internal/app/library_fetch.go b/internal/app/library_fetch.go new file mode 100644 index 0000000..6197e20 --- /dev/null +++ b/internal/app/library_fetch.go @@ -0,0 +1,98 @@ +package app + +import ( + "context" + "errors" + "fmt" + "time" +) + +func (d *Downloader) fetchAllDramas(ctx context.Context, sourceFilter string) ([]Drama, error) { + if sourceFilter != "" && canonicalProviderSource(sourceFilter) != sourceHongguo { + return nil, errors.New("此版本仅支持红果站源") + } + if more, _ := ctx.Value(libraryMoreKey{}).(bool); more { + return d.fetchMoreLibrary(ctx, sourceFilter) + } + category, _ := ctx.Value(libraryCategoryKey{}).(string) + fmt.Println("正在获取红果剧库列表...") + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + dramas, err := d.fetchHongguoCategoryDramas(ctx, category) + dramas = onlyHongguoDramas(dramas) + if len(dramas) == 0 && err == nil && !hongguoCatalogInitialized(d.hongguoCatalogSnapshot()) { + err = errors.New("未返回可识别的视频数据") + } + reportLibraryProgress(ctx, sourceHongguo, dramas, err, true) + fmt.Printf(" 剧库获取完成:%d 部\n", len(dramas)) + if err != nil { + fmt.Printf(" 红果获取失败: %v\n", publicError(err)) + return dramas, &libraryLoadError{failures: map[string]error{sourceHongguo: err}} + } + return dramas, nil +} + +func (d *Downloader) GetDramaChapters(ctx context.Context, seriesID string) (string, []Chapter, error) { + _, sourceID, valid := splitProviderDramaID(seriesID) + if !valid { + return "", nil, errors.New("仅支持红果剧集 ID,可填写 hongguo:数字ID 或数字ID") + } + title, chapters, err := d.fetchHongguoChapters(ctx, sourceID) + return title, uniqueChapters(chapters), err +} + +func uniqueChapters(chapters []Chapter) []Chapter { + seen := map[string]bool{} + var out []Chapter + for i, ch := range chapters { + key := ch.ID + if key == "" { + key = ch.VideoURL + } + if key == "" { + key = fmt.Sprintf("idx_%d", i) + } + if seen[key] { + continue + } + seen[key] = true + out = append(out, ch) + } + return out +} + +func (d *Downloader) BuildDramaTasks(ctx context.Context, drama Drama) ([]Task, error) { + return d.buildDramaTasksInDirectory(ctx, drama, "") +} + +func (d *Downloader) buildDramaTasksInDirectory(ctx context.Context, drama Drama, existingDirectory string) ([]Task, error) { + var valid bool + drama, valid = normalizeHongguoDrama(drama) + if !valid { + return nil, errors.New("此版本仅支持红果剧集") + } + title, chapters, err := d.GetDramaChapters(ctx, drama.ID) + if err != nil { + return nil, err + } + if len(chapters) == 0 { + return nil, nil + } + drama = d.hongguoCachedDrama(drama) + dramaDir, err := d.downloadDramaDirectory(drama, title, existingDirectory) + if err != nil { + return nil, err + } + var tasks []Task + for i, ch := range chapters { + ep := ch.EpisodeString(i + 1) + epNum := padEpisode(ep) + fileName := safeFilename(epNum + ".mp4") + outPath, err := safeJoin(dramaDir, fileName) + if err != nil { + return nil, err + } + tasks = append(tasks, Task{DramaID: drama.ID, DramaTitle: title, Chapter: ch, Index: i + 1, Total: len(chapters), OutPath: outPath, ReleaseStatus: dramaReleaseStatus(drama)}) + } + return tasks, nil +} diff --git a/internal/app/library_metadata.go b/internal/app/library_metadata.go new file mode 100644 index 0000000..8f279df --- /dev/null +++ b/internal/app/library_metadata.go @@ -0,0 +1,252 @@ +package app + +import ( + "context" + "errors" + "fmt" + "net/http" + "sort" + "strings" + "sync" + "time" +) + +const sortMetadataVersion = 1 +const sortMetadataBatchSize = 40 +const sortMetadataRetryDelay = 5 * time.Minute +const sortMetadataMissingRetryDelay = 24 * time.Hour + +type sortMetadataState struct { + Version int `json:"version"` + CheckedAt time.Time `json:"checkedAt"` + Pending bool `json:"pending,omitempty"` + CoverChecked bool `json:"coverChecked,omitempty"` + RetryAt time.Time `json:"retryAt,omitempty"` + LastError string `json:"lastError,omitempty"` +} + +type libraryRowsKey struct{} +type libraryPriorityKey struct{} +type libraryMetadataProgressKey struct{} + +type libraryMetadataProgress struct { + Running bool `json:"running"` + Checked int `json:"checked"` + Total int `json:"total"` + Updated int `json:"updated"` + Failed int `json:"failed"` +} + +func supportsSortMetadata(drama Drama) bool { + switch dramaProvider(drama) { + case sourceHongguo: + return true + } + return false +} + +func needsSortMetadata(drama Drama) bool { + return supportsSortMetadata(drama) && (drama.SortMetadata == nil || drama.SortMetadata.Version != sortMetadataVersion || needsHongguoCoverAddress(drama)) +} + +func sortMetadataRetryAt(state *sortMetadataState, now time.Time) time.Time { + if state == nil || state.Version != 0 { + return time.Time{} + } + if !state.RetryAt.IsZero() { + return state.RetryAt + } + if state.CheckedAt.IsZero() { + return time.Time{} + } + return state.CheckedAt.Add(sortMetadataRetryDelay) +} + +func sortMetadataFailureDelay(err error) time.Duration { + var status *httpStatusError + if errors.As(err, &status) && (status.status == http.StatusNotFound || status.status == http.StatusGone) { + return sortMetadataMissingRetryDelay + } + text := strings.ToLower(err.Error()) + if strings.Contains(text, "无效的") || strings.Contains(text, "未返回所请求") || strings.Contains(text, "返回了其他剧集") || strings.Contains(text, "与请求剧集不符") { + return sortMetadataMissingRetryDelay + } + return sortMetadataRetryDelay +} + +func failedSortMetadataState(drama Drama, err error, now time.Time) *sortMetadataState { + state := &sortMetadataState{CheckedAt: now, RetryAt: now.Add(sortMetadataFailureDelay(err)), LastError: truncate(publicError(err).Error(), 300)} + if drama.SortMetadata != nil { + state.CoverChecked = drama.SortMetadata.CoverChecked + } + return state +} + +func needsHongguoCoverAddress(drama Drama) bool { + return dramaProvider(drama) == sourceHongguo && bestDramaCover(drama) == "" && + (drama.SortMetadata == nil || !drama.SortMetadata.CoverChecked) +} + +func selectSortMetadataBatch(dramas []Drama, source string, priority []string, now time.Time) []Drama { + positions := make(map[string]int, len(priority)) + for index, id := range priority { + if _, found := positions[id]; !found { + positions[id] = index + } + } + var candidates []Drama + seen := map[string]bool{} + for _, drama := range dramas { + if seen[drama.ID] || !matchesSourceFilter(dramaProvider(drama), source) || !needsSortMetadata(drama) { + continue + } + seen[drama.ID] = true + if retryAt := sortMetadataRetryAt(drama.SortMetadata, now); !retryAt.IsZero() && now.Before(retryAt) { + continue + } + candidates = append(candidates, drama) + } + sort.SliceStable(candidates, func(i, j int) bool { + left, leftOK := positions[candidates[i].ID] + right, rightOK := positions[candidates[j].ID] + if leftOK != rightOK { + return leftOK + } + if leftOK && left != right { + return left < right + } + var leftTime, rightTime time.Time + if candidates[i].SortMetadata != nil { + leftTime = candidates[i].SortMetadata.CheckedAt + } + if candidates[j].SortMetadata != nil { + rightTime = candidates[j].SortMetadata.CheckedAt + } + return leftTime.Before(rightTime) + }) + if len(candidates) > sortMetadataBatchSize { + candidates = candidates[:sortMetadataBatchSize] + } + return candidates +} + +func sortMetadataRemaining(dramas []Drama) map[string]int { + counts := map[string]int{"": 0, "hongguo": 0} + for _, drama := range dramas { + if !needsSortMetadata(drama) { + continue + } + source := dramaProvider(drama) + counts[source]++ + counts[""]++ + } + return counts +} + +func reportMetadataProgress(ctx context.Context, progress libraryMetadataProgress) { + if callback, ok := ctx.Value(libraryMetadataProgressKey{}).(func(libraryMetadataProgress)); ok { + callback(progress) + } +} + +func (d *Downloader) backfillSortMetadata(ctx context.Context, source string) ([]Drama, map[string]error) { + rows, _ := ctx.Value(libraryRowsKey{}).([]Drama) + priority, _ := ctx.Value(libraryPriorityKey{}).([]string) + batch := selectSortMetadataBatch(rows, source, priority, time.Now()) + progress := libraryMetadataProgress{Running: true, Total: len(batch)} + reportMetadataProgress(ctx, progress) + defer func() { progress.Running = false; reportMetadataProgress(ctx, progress) }() + failures := map[string]error{} + if len(batch) == 0 { + return nil, failures + } + workCtx, cancel := context.WithTimeout(ctx, 3*time.Minute) + defer cancel() + type result struct { + patch Drama + previous Drama + err error + } + jobs := make(chan Drama, len(batch)) + results := make(chan result, len(batch)) + for _, drama := range batch { + jobs <- drama + } + close(jobs) + workers := d.cfg.RequestConcurrency + if workers < 1 { + workers = 1 + } + if workers > 3 { + workers = 3 + } + var group sync.WaitGroup + for worker := 0; worker < workers; worker++ { + group.Add(1) + go func() { + defer group.Done() + for drama := range jobs { + if workCtx.Err() != nil { + return + } + itemCtx, stop := context.WithTimeout(workCtx, 40*time.Second) + patch, err := d.fetchDramaSortMetadata(itemCtx, drama) + stop() + now := time.Now() + patch.SortMetadata = &sortMetadataState{CheckedAt: now} + if err == nil { + patch.SortMetadata.Version = sortMetadataVersion + patch.SortMetadata.CoverChecked = true + } else { + patch.SortMetadata = failedSortMetadataState(drama, err, now) + } + results <- result{patch: patch, previous: drama, err: err} + } + }() + } + go func() { group.Wait(); close(results) }() + var patches []Drama + for result := range results { + patch := result.patch + provider := dramaProvider(result.previous) + if result.err != nil { + progress.Failed++ + if failures[provider] == nil { + failures[provider] = fmt.Errorf("部分历史资料未补齐,可稍后重试: %w", result.err) + } + } + if patch.OnlineDate != "" && patch.OnlineDate != result.previous.OnlineDate || patch.Heat != "" && patch.Heat != result.previous.Heat || patch.Views != "" && patch.Views != result.previous.Views || bestDramaCover(patch) != "" && bestDramaCover(patch) != bestDramaCover(result.previous) { + progress.Updated++ + } + patches = append(patches, patch) + progress.Checked++ + reportLibraryProgress(ctx, provider, []Drama{patch}, nil, false) + reportMetadataProgress(ctx, progress) + } + if progress.Checked < len(batch) { + for _, drama := range batch { + provider := dramaProvider(drama) + if failures[provider] == nil { + failures[provider] = errors.New("本批补齐已暂停,剩余条目可再次手动补齐") + } + } + } + return patches, failures +} + +func (d *Downloader) fetchMoreLibrary(ctx context.Context, source string) ([]Drama, error) { + category, _ := ctx.Value(libraryCategoryKey{}).(string) + patches, failures := d.backfillSortMetadata(ctx, source) + if matchesSourceFilter(sourceHongguo, source) { + items, err := d.fetchHongguoCategoryDramas(ctx, category) + if err != nil { + failures[sourceHongguo] = errors.Join(failures[sourceHongguo], err) + } + reportLibraryProgress(ctx, sourceHongguo, items, failures[sourceHongguo], true) + patches = mergeSourceDramas(patches, items, nil, sourceHongguo) + } + if len(failures) > 0 { + return patches, &libraryLoadError{failures: failures} + } + return patches, nil +} diff --git a/internal/app/library_metadata_queue.go b/internal/app/library_metadata_queue.go new file mode 100644 index 0000000..52b54e8 --- /dev/null +++ b/internal/app/library_metadata_queue.go @@ -0,0 +1,185 @@ +package app + +import ( + "context" + "time" +) + +func (a *UIApp) newSortMetadataDramasLocked(items []Drama) []Drama { + known := make(map[string]bool, len(a.dramas)) + for _, drama := range a.dramas { + known[drama.ID] = true + } + var fresh []Drama + for _, drama := range items { + if drama.ID != "" && !known[drama.ID] { + known[drama.ID] = true + if needsSortMetadata(drama) { + fresh = append(fresh, drama) + } + } + } + return fresh +} + +func (a *UIApp) enqueueHistoricalMetadataLocked(source string, priority []string) { + var available []Drama + for _, drama := range a.dramas { + if !a.metadataPending[drama.ID] { + available = append(available, drama) + } + } + a.enqueueSortMetadataLocked(selectSortMetadataBatch(available, source, priority, time.Now()), true) +} + +func (a *UIApp) enqueueSortMetadataLocked(items []Drama, prioritize bool) { + if a.metadataClosed || len(items) == 0 { + return + } + if a.metadataPending == nil { + a.metadataPending = map[string]bool{} + } + var added []string + for _, drama := range items { + if drama.ID == "" || !needsSortMetadata(drama) || a.metadataPending[drama.ID] { + continue + } + a.metadataPending[drama.ID] = true + added = append(added, drama.ID) + } + if len(added) == 0 { + return + } + + for index, drama := range a.dramas { + if !a.metadataPending[drama.ID] || drama.SortMetadata != nil && drama.SortMetadata.Pending { + continue + } + state := sortMetadataState{Pending: true} + if drama.SortMetadata != nil { + state = *drama.SortMetadata + state.Pending = true + } + a.dramas[index].SortMetadata = &state + } + if prioritize { + a.metadataQueue = append(added, a.metadataQueue...) + } else { + a.metadataQueue = append(a.metadataQueue, added...) + } + if !a.libraryMetadata.Running { + a.libraryMetadata = libraryMetadataProgress{Running: true} + } + a.libraryMetadata.Total += len(added) + a.libraryDirty = true + a.libraryRevision++ + if a.metadataDone == nil { + ctx, cancel := context.WithCancel(context.Background()) + a.metadataCancel = cancel + a.metadataDone = make(chan struct{}) + go a.runSortMetadataQueue(ctx, a.metadataDone) + } +} + +func (a *UIApp) runSortMetadataQueue(ctx context.Context, done chan struct{}) { + + ctx = context.WithValue(ctx, backgroundCatalogKey{}, true) + for { + a.mu.Lock() + if ctx.Err() != nil || len(a.metadataQueue) == 0 { + a.mu.Unlock() + a.persistLibrary() + a.mu.Lock() + + if ctx.Err() == nil && len(a.metadataQueue) > 0 { + a.mu.Unlock() + continue + } + a.libraryMetadata.Running = false + a.libraryRevision++ + a.metadataCancel() + a.metadataCancel = nil + a.metadataDone = nil + a.metadataQueue = nil + close(done) + a.mu.Unlock() + return + } + if a.libraryLoading != nil { + a.mu.Unlock() + timer := time.NewTimer(500 * time.Millisecond) + select { + case <-ctx.Done(): + case <-timer.C: + } + timer.Stop() + continue + } + id := a.metadataQueue[0] + a.metadataQueue = a.metadataQueue[1:] + var previous Drama + for _, drama := range a.dramas { + if drama.ID == id { + previous = drama + break + } + } + a.mu.Unlock() + itemCtx, cancel := context.WithTimeout(ctx, 40*time.Second) + patch, err := a.downloader.fetchDramaSortMetadata(itemCtx, previous) + cancel() + if err != nil && ctx.Err() == nil { + a.downloader.recordDiagnostic(diagnosticEvent{Event: "metadata.failed", Source: dramaProvider(previous), DramaID: previous.ID, DramaTitle: previous.DisplayTitle(), Message: err.Error()}) + } + patch.SortMetadata = &sortMetadataState{CheckedAt: time.Now()} + if err == nil { + patch.SortMetadata.Version = sortMetadataVersion + patch.SortMetadata.CoverChecked = true + } + normalizeDramaCover(&patch) + a.mu.Lock() + if ctx.Err() != nil { + + a.mu.Unlock() + continue + } + delete(a.metadataPending, id) + for index, drama := range a.dramas { + if drama.ID != id { + continue + } + if patch.OnlineDate != "" && patch.OnlineDate != drama.OnlineDate || patch.Heat != "" && patch.Heat != drama.Heat || patch.Views != "" && patch.Views != drama.Views || bestDramaCover(patch) != "" && bestDramaCover(patch) != bestDramaCover(drama) { + a.libraryMetadata.Updated++ + } + + patch.Title, patch.Name = drama.Title, drama.Name + a.dramas[index] = mergeDramaMetadata(patch, drama) + break + } + a.libraryMetadata.Checked++ + if err != nil { + a.libraryMetadata.Failed++ + } + a.libraryDirty = true + a.libraryRevision++ + persist := time.Since(a.libraryLastSave) >= 3*time.Second + if persist { + a.libraryLastSave = time.Now() + } + a.mu.Unlock() + if persist { + a.persistLibrary() + } + } +} + +func (a *UIApp) stopSortMetadata() { + a.mu.Lock() + a.metadataClosed = true + cancel, done := a.metadataCancel, a.metadataDone + a.mu.Unlock() + if cancel != nil { + cancel() + <-done + } +} diff --git a/internal/app/library_metadata_queue_test.go b/internal/app/library_metadata_queue_test.go new file mode 100644 index 0000000..a787aa9 --- /dev/null +++ b/internal/app/library_metadata_queue_test.go @@ -0,0 +1,243 @@ +package app + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" +) + +func waitSortMetadataQueue(t *testing.T, app *UIApp) { + t.Helper() + app.mu.Lock() + done := app.metadataDone + app.mu.Unlock() + if done != nil { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("metadata queue did not finish") + } + } +} + +func TestPrimaryCatalogReturnsBeforeOptionalMetadata(t *testing.T) { + for _, source := range []string{sourceHongguo} { + t.Run(source, func(t *testing.T) { + id := "7615465407347952664" + started, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + var calls atomic.Int32 + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.Context().Value(backgroundCatalogKey{}) != true { + t.Error("optional detail did not use background priority") + } + once.Do(func() { close(started) }) + select { + case <-release: + case <-request.Context().Done(): + return nil, request.Context().Err() + } + switch request.URL.Path { + case "/detail": + if request.URL.Query().Get("series_id") != id { + t.Error("historical entry was automatically fetched") + } + return rankingHTTPResponse(request, 200, sortDetailFixture(id, "1773662280")), nil + case "/novel/player/video_detail/v1/": + return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":%q,"series_title":"详情标题","hot_score":20019,"series_play_cnt":10019}}}`, id)), nil + default: + t.Errorf("unexpected metadata request: %s", request.URL.Path) + return rankingHTTPResponse(request, 404, ""), nil + } + }) + old := Drama{ID: providerDramaID(source, "7498612570866076734"), Source: source, Title: "历史条目"} + fresh := Drama{ID: providerDramaID(source, id), Source: source, Title: "列表标题", Heat: "20019", Views: "0", Cover: coverAddressFixture} + app := &UIApp{downloader: d, cfg: d.cfg, dramas: []Drama{old}, librarySources: map[string]librarySourceState{}, libraryLoading: make(chan struct{})} + t.Cleanup(app.stopSortMetadata) + app.acceptLibraryProgress(source, []Drama{old, fresh}, nil, false) + app.mu.Lock() + if calls.Load() != 0 || len(app.dramas) != 2 || app.dramas[1].Heat != fresh.Heat || app.dramas[1].Views != "0" { + t.Error("primary fields did not return before optional requests") + } + close(app.libraryLoading) + app.libraryLoading = nil + app.mu.Unlock() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("new entry was not scheduled automatically") + } + + app.acceptLibraryProgress(source, []Drama{fresh, old}, nil, true) + writer := httptest.NewRecorder() + app.handleDramas(writer, httptest.NewRequest("GET", "/api/ui/dramas", nil)) + var snapshot struct { + Loading bool `json:"loading"` + Metadata libraryMetadataProgress `json:"metadata"` + } + if json.Unmarshal(writer.Body.Bytes(), &snapshot) != nil || snapshot.Loading || !snapshot.Metadata.Running || snapshot.Metadata.Total != 1 { + t.Fatal("optional metadata blocked the library or duplicated work") + } + close(release) + waitSortMetadataQueue(t, app) + cache, err := readLibraryCache(d.cfg.dataDirectory()) + if err != nil { + t.Fatal(err) + } + for _, row := range cache.Dramas { + if row.ID == old.ID && row.SortMetadata != nil { + t.Error("ordinary catalog refresh enrolled historical data") + } + wantViews := "10019次播放" + if source == sourceHongguo { + wantViews = "0" + } + if row.ID == fresh.ID && (row.OnlineDate != "2026-03-16" || row.Views != wantViews || row.Heat != fresh.Heat || row.Title != "列表标题" || row.SortMetadata == nil || row.SortMetadata.Version != sortMetadataVersion || row.SortMetadata.Pending) { + t.Errorf("asynchronous metadata was not merged correctly: %+v", row) + } + } + if calls.Load() != 1 { + t.Fatalf("duplicate detail requests: %d", calls.Load()) + } + }) + } +} + +func TestResumeOnlyPreviouslyQueuedMetadata(t *testing.T) { + var calls atomic.Int32 + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.URL.Path != "/detail" || request.URL.Query().Get("series_id") != "7000000000000000002" { + t.Errorf("unexpected resource: %s", request.URL.Path) + } + return rankingHTTPResponse(request, 200, sortDetailFixture("7000000000000000002", "0")), nil + }) + cache := libraryCache{Dramas: []Drama{ + {ID: "hongguo:7000000000000000001", Source: sourceHongguo, Title: "历史条目"}, + {ID: "hongguo:7000000000000000002", Source: sourceHongguo, Title: "新条目", Heat: "0", Views: "0", Cover: coverAddressFixture, SortMetadata: &sortMetadataState{Pending: true}}, + }} + if err := writeLibraryCache(d.cfg.dataDirectory(), cache); err != nil { + t.Fatal(err) + } + app := &UIApp{downloader: d, cfg: d.cfg} + t.Cleanup(app.stopSortMetadata) + app.loadLibrary() + waitSortMetadataQueue(t, app) + if calls.Load() != 1 { + t.Fatalf("resume requested %d resources, want one pending detail", calls.Load()) + } + app.mu.Lock() + defer app.mu.Unlock() + if app.dramas[0].SortMetadata != nil || app.dramas[1].OnlineDate != "" || app.dramas[1].Views != "0" || app.dramas[1].SortMetadata.Version != sortMetadataVersion { + t.Fatal("resume changed history, lost zero, or invented an empty source date") + } +} + +func TestPrimarySortFieldsNeedNoAdditionalInterfacesWhenComplete(t *testing.T) { + hongguo := hongguoDramaFromAny(map[string]any{"series_id": "7615465407347952664", "series_title": "列表剧", "series_cover": coverAddressFixture, "first_visible_time": "1773662280", "hot_score": "20019", "series_play_cnt": "0"}, "测试") + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + t.Errorf("complete primary data triggered an optional request: %s", request.URL.Path) + return rankingHTTPResponse(request, 404, ""), nil + }) + for _, drama := range []Drama{hongguo} { + if drama.OnlineDate != "2026-03-16" || drama.Views != "0" || !hasCompleteSortMetadata(drama) { + t.Errorf("primary sort fields were lost: %+v", drama) + } + if _, err := d.fetchDramaSortMetadata(context.Background(), drama); err != nil { + t.Fatal(err) + } + } + if hasSortMetric("未知") || hasSortMetric("-1") { + t.Fatal("unknown metrics were misclassified as available sorting data") + } +} + +func TestBackgroundRequestsYieldToForeground(t *testing.T) { + limiter := newRequestLimiter(1, time.Nanosecond) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + request := httptest.NewRequest("GET", "https://hongguoduanju.com/search/test", nil) + release, err := limiter.acquire(ctx, request) + if err != nil { + t.Fatal(err) + } + background := make(chan func(), 1) + foreground := make(chan func(), 1) + go func() { + unlock, _ := limiter.acquire(context.WithValue(ctx, backgroundCatalogKey{}, true), request) + background <- unlock + }() + go func() { + unlock, _ := limiter.acquire(ctx, request) + foreground <- unlock + }() + + for { + limiter.mu.Lock() + waiting := limiter.foregroundWaiting + limiter.mu.Unlock() + if waiting > 0 { + break + } + if ctx.Err() != nil { + t.Fatal("foreground waiter did not register") + } + time.Sleep(time.Millisecond) + } + release() + select { + case unlock := <-foreground: + if unlock == nil { + t.Fatal("foreground timed out") + } + select { + case <-background: + t.Fatal("background ran alongside a waiting foreground request") + default: + } + unlock() + case <-background: + t.Fatal("optional metadata overtook foreground work") + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case unlock := <-background: + if unlock == nil { + t.Fatal("background did not resume") + } + unlock() + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } +} + +func TestOptionalDetailPermissionFailureDoesNotBlockForeground(t *testing.T) { + limiter := newRequestLimiter(1, time.Nanosecond) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + foreground := httptest.NewRequest("GET", "https://hongguoduanju.com/search/test", nil).WithContext(ctx) + background := foreground.WithContext(context.WithValue(ctx, backgroundCatalogKey{}, true)) + limiter.observe(background, &http.Response{StatusCode: http.StatusForbidden, Header: http.Header{}}) + if release, err := limiter.acquire(background.Context(), background); err == nil { + release() + t.Fatal("repeated forbidden details must cool down") + } + release, err := limiter.acquire(ctx, foreground) + if err != nil { + t.Fatal("an optional detail's permissions blocked public search", err) + } + release() + limiter.observe(background, &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{"Retry-After": {"60"}}}) + if release, err := limiter.acquire(ctx, foreground); err == nil { + release() + t.Fatal("site rate limits must still be respected") + } +} diff --git a/internal/app/library_metadata_test.go b/internal/app/library_metadata_test.go new file mode 100644 index 0000000..73ddf71 --- /dev/null +++ b/internal/app/library_metadata_test.go @@ -0,0 +1,185 @@ +package app + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func sortDetailFixture(id, stamp string) string { + return `` +} + +func TestSortDetailDatesAndIdentity(t *testing.T) { + id := "7615465407347952664" + drama, err := parseHongguoSortDetail(sortDetailFixture(id, "1773662280"), id) + if err != nil || drama.OnlineDate != "2026-03-16" { + t.Fatalf("first-visible date: %q, %v", drama.OnlineDate, err) + } + if _, err = parseHongguoSortDetail(sortDetailFixture("7498612570866076734", "1773662280"), id); err == nil { + t.Fatal("accepted another drama's time") + } + drama, err = parseHongguoSortDetail(sortDetailFixture(id, "0"), id) + if err != nil || drama.OnlineDate != "" { + t.Fatal("creation time must not replace missing first-visible time") + } + for _, value := range []string{"", "0", "0001-01-01", "1970-01-01", "2101-01-01", "2026-02-31", "not a date"} { + if got := providerReleaseDate(value); got != "" { + t.Errorf("accepted date %q", value) + } + } + if normalizeViews("10019") != "10019次播放" { + t.Fatal("play counts must retain precision") + } +} + +func TestSortMetadataSelectionAndResume(t *testing.T) { + now := time.Now() + var rows []Drama + for index := 0; index < 60; index++ { + rows = append(rows, Drama{ID: fmt.Sprintf("hongguo:%019d", index+1), Source: sourceHongguo}) + } + rows[1].SortMetadata = &sortMetadataState{Version: sortMetadataVersion, CheckedAt: now, CoverChecked: true} + rows[2].SortMetadata = &sortMetadataState{CheckedAt: now} + rows = append(rows, Drama{ID: "huangdou:other", Source: "huangdou"}) + batch := selectSortMetadataBatch(rows, sourceHongguo, []string{rows[59].ID, rows[59].ID, rows[2].ID, "huangdou:other"}, now) + if len(batch) != sortMetadataBatchSize || batch[0].ID != rows[59].ID { + t.Fatal("did not prioritize filtered entries within the batch limit") + } + for _, row := range batch { + if row.ID == rows[1].ID || row.ID == rows[2].ID || row.Source != sourceHongguo { + t.Fatal("selected completed, cooling-down or other-source entries") + } + } + if got := selectSortMetadataBatch(rows, "huangdou", nil, now); len(got) != 0 { + t.Fatal("source filter was not respected") + } + if remaining := sortMetadataRemaining(rows); remaining["hongguo"] != 59 || remaining["huangdou"] != 0 || remaining[""] != 59 { + t.Fatal(remaining) + } +} + +func TestManualBackfillPersistsPartialResultsAndContinuesAfterCatalogEnd(t *testing.T) { + first, bad, unknown := "7615465407347952664", "7498612570866076734", "7677914379060251673" + var calls atomic.Int32 + var fail atomic.Bool + fail.Store(true) + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + calls.Add(1) + switch request.URL.Path { + case "/detail": + id := request.URL.Query().Get("series_id") + stamp := "1773662280" + if id == unknown { + stamp = "0" + } + return rankingHTTPResponse(request, 200, sortDetailFixture(id, stamp)), nil + case "/novel/player/video_detail/v1/": + var body struct { + ID string `json:"series_id"` + } + json.NewDecoder(request.Body).Decode(&body) + if body.ID == bad && fail.Load() { + return rankingHTTPResponse(request, 503, "temporary failure"), nil + } + return rankingHTTPResponse(request, 200, fmt.Sprintf(`{"data":{"video_data":{"series_id_str":%q,"series_title":"测试剧","hot_score":20019,"series_play_cnt":10019}}}`, body.ID)), nil + default: + t.Errorf("backfill requested a non-detail resource: %s", request.URL.Path) + return rankingHTTPResponse(request, 404, ""), nil + } + }) + client := d.hongguoClient() + client.mu.Lock() + for _, genre := range hongguoAppGenres { + client.state.Feeds[genre.key] = hongguoCatalogCursor{Initialized: true, Exhausted: true} + } + client.mu.Unlock() + a := &UIApp{downloader: d, cfg: d.cfg, libraryAttempted: true, librarySources: map[string]librarySourceState{}, libraryApp: d.hongguoCatalogSnapshot(), dramas: []Drama{ + {ID: "hongguo:" + first, Source: sourceHongguo, Title: "原有标题", Views: "0"}, + {ID: "hongguo:" + bad, Source: sourceHongguo, Title: "暂时失败", Heat: "800"}, + {ID: "hongguo:" + unknown, Source: sourceHongguo, Title: "站点日期未知"}, + }} + t.Cleanup(a.stopSortMetadata) + plain := httptest.NewRecorder() + a.handleDramas(plain, httptest.NewRequest("GET", "/api/ui/dramas", nil)) + if calls.Load() != 0 { + t.Fatal("ordinary library reads must not backfill") + } + before := a.librarySnapshotLocked(0) + if before["hasMoreBySource"].(map[string]bool)[sourceHongguo] != true { + t.Fatal("backfill must remain available after the catalog is exhausted") + } + writer := httptest.NewRecorder() + a.handleDramas(writer, httptest.NewRequest("GET", "/api/ui/dramas?more=1&source=hongguo&priority=hongguo:"+first, nil)) + if writer.Code != 200 { + t.Fatal(writer.Body.String()) + } + waitSortMetadataQueue(t, a) + a.mu.Lock() + rows := append([]Drama(nil), a.dramas...) + progress := a.libraryMetadata + a.mu.Unlock() + if progress.Checked != 3 || progress.Failed != 1 { + t.Fatalf("unexpected progress: %+v", progress) + } + for _, row := range rows { + switch row.ID { + case "hongguo:" + first: + if row.OnlineDate != "2026-03-16" || row.Heat != "20019" || row.Views != "10019次播放" || row.SortMetadata.Version != sortMetadataVersion { + t.Fatal("fresh sorting fields were not merged") + } + case "hongguo:" + bad: + if row.OnlineDate == "" || row.Heat != "800" || row.SortMetadata.Version != 0 { + t.Fatal("partial success must survive a failed metrics request") + } + case "hongguo:" + unknown: + if row.OnlineDate != "" || row.SortMetadata.Version != sortMetadataVersion { + t.Fatal("a valid response with no date must not be retried forever or invent a date") + } + } + } + saved, err := readLibraryCache(d.cfg.dataDirectory()) + if err != nil { + t.Fatal(err) + } + if batch := selectSortMetadataBatch(saved.Dramas, sourceHongguo, nil, time.Now()); len(batch) != 0 { + t.Fatal("restart lost completed checks or failure cooldown") + } + for index, row := range saved.Dramas { + if row.ID == "hongguo:"+bad { + saved.Dramas[index].SortMetadata = &sortMetadataState{CheckedAt: time.Now().Add(-sortMetadataRetryDelay - time.Second)} + } + } + fail.Store(false) + previous := calls.Load() + patches, failures := d.backfillSortMetadata(withKnownHongguoDramas(context.Background(), saved.Dramas), sourceHongguo) + if len(failures) != 0 || len(patches) != 1 || patches[0].ID != "hongguo:"+bad || calls.Load()-previous != 1 { + t.Fatal("retry did not resume only the failed entry") + } +} + +func TestManualMetadataPriorityValidation(t *testing.T) { + a := &UIApp{} + for _, path := range []string{ + "/api/ui/dramas?more=1&source=invalid", + "/api/ui/dramas?more=1&refresh=1", + "/api/ui/dramas?update=1&refresh=1", + "/api/ui/dramas?update=1&more=1", + "/api/ui/dramas?update=1&source=invalid", + "/api/ui/dramas?update=1&priority=" + strings.Repeat("hongguo:700001,", sortMetadataBatchSize), + "/api/ui/dramas?update=1&priority=hongguo:" + strings.Repeat("1", 121), + "/api/ui/dramas?more=1&priority=" + strings.Repeat("id,", sortMetadataBatchSize), + } { + writer := httptest.NewRecorder() + a.handleDramas(writer, httptest.NewRequest("GET", path, nil)) + if writer.Code != 400 { + t.Fatalf("invalid request was not rejected: %d", writer.Code) + } + } +} diff --git a/internal/app/library_progress.go b/internal/app/library_progress.go new file mode 100644 index 0000000..fec7e9e --- /dev/null +++ b/internal/app/library_progress.go @@ -0,0 +1,371 @@ +package app + +import ( + "context" + "errors" + "fmt" + "net/http" + "sort" + "strings" + "time" +) + +type librarySourceState struct { + Status string `json:"status"` + Count int `json:"count"` + Error string `json:"error,omitempty"` + Phase string `json:"phase,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func (state librarySourceState) withPhase(phase string) librarySourceState { + phase = strings.TrimSpace(phase) + if state.Status == "loading" { + state.Phase = phase + } + return state +} + +type libraryProgressKey struct{} +type libraryCategoryKey struct{} +type libraryProgressFunc func(string, []Drama, error, bool) error + +type libraryLoadMode int + +const ( + libraryLoadRefresh libraryLoadMode = iota + libraryLoadMore + libraryLoadUpdate +) + +func reportLibraryProgress(ctx context.Context, source string, items []Drama, err error, done bool) error { + if callback, ok := ctx.Value(libraryProgressKey{}).(libraryProgressFunc); ok && (done || len(items) > 0) { + return callback(source, items, err, done) + } + return nil +} + +func (a *UIApp) setLibraryPhaseLocked(phase string) { + phase = strings.TrimSpace(phase) + a.libraryLoadingPhase = phase + if a.librarySources == nil { + a.librarySources = map[string]librarySourceState{} + } + for source, state := range a.librarySources { + if state.Status == "loading" { + a.librarySources[source] = state.withPhase(phase) + } + } +} + +func (a *UIApp) handleLibraryCancel(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "method not allowed"}) + return + } + a.mu.Lock() + cancel := a.libraryCancel + running := a.libraryLoading != nil + if running { + a.setLibraryPhaseLocked("正在停止,已保留已入库内容") + a.libraryRevision++ + } + resp := a.librarySnapshotForSourceLocked(r.Context(), 0) + a.mu.Unlock() + if cancel != nil { + cancel() + } + resp["canceled"] = running + writeJSON(w, http.StatusOK, resp) +} + +func (a *UIApp) startLibraryRefreshLocked(source string) { + a.startLibraryLoadLocked(source, libraryLoadRefresh, nil, "") +} + +func (a *UIApp) startLibraryLoadLocked(source string, mode libraryLoadMode, priority []string, category string, scopes ...context.Context) { + category = strings.TrimSpace(category) + base := context.Background() + if len(scopes) > 0 && sourceScope(scopes[0]) != nil { + base = context.WithValue(base, accountSourceContextKey{}, sourceScope(scopes[0])) + } + if mode == libraryLoadUpdate || mode == libraryLoadMore { + if source == "" && sourceScopeRestricted(base) { + for _, allowed := range sourceScope(base).Sources { + a.enqueueHistoricalMetadataLocked(allowed, priority) + } + } else { + a.enqueueHistoricalMetadataLocked(source, priority) + } + } + more := mode == libraryLoadMore + if more { + if !matchesSourceFilter(sourceHongguo, source) || !hongguoCatalogCategoryHasMore(a.libraryApp, category) { + return + } + source = sourceHongguo + } + previousCount := len(a.dramas) + a.libraryAttempted = true + a.libraryLoading = make(chan struct{}) + a.libraryMore = more + a.libraryLoadingSource = source + a.libraryLoadingPhase = "等待开始" + if a.librarySources == nil { + a.librarySources = map[string]librarySourceState{} + } + for _, provider := range []string{sourceHongguo} { + if matchesSourceFilter(provider, source) { + state := a.librarySources[provider] + state.Status = "loading" + state.Error = "" + state.Phase = "等待开始" + a.librarySources[provider] = state + } + } + a.libraryRevision++ + ctx, cancel := context.WithTimeout(base, 12*time.Minute) + a.libraryCancel = cancel + ctx = context.WithValue(ctx, libraryProgressKey{}, libraryProgressFunc(a.acceptLibraryProgress)) + ctx = context.WithValue(ctx, libraryMoreKey{}, more) + ctx = context.WithValue(ctx, libraryUpdateKey{}, mode == libraryLoadUpdate) + if category != "" { + ctx = context.WithValue(ctx, libraryCategoryKey{}, category) + } + ctx = withKnownHongguoDramas(ctx, a.dramas) + go func() { + defer cancel() + var loadErr error + a.mu.Lock() + a.setLibraryPhaseLocked("保存当前剧库") + a.libraryRevision++ + a.mu.Unlock() + if err := a.persistLibraryIfDirty(); err != nil { + loadErr = err + } else if more { + a.mu.Lock() + a.setLibraryPhaseLocked("继续加载目录") + a.libraryRevision++ + a.mu.Unlock() + items, err := a.downloader.fetchHongguoCategoryDramas(ctx, category) + loadErr = errors.Join(err, reportLibraryProgress(ctx, sourceHongguo, items, err, true)) + } else { + a.mu.Lock() + a.setLibraryPhaseLocked("读取站源目录") + a.libraryRevision++ + a.mu.Unlock() + _, loadErr = a.downloader.fetchAllDramas(ctx, source) + } + a.mu.Lock() + if ctx.Err() == context.Canceled { + loadErr = nil + a.markLibrarySourcesCanceledLocked(base, source) + } else if loadErr != nil && a.libraryLoading != nil { + a.markLibrarySourcesFailedLocked(base, source, loadErr) + } + a.libraryDirty = true + a.libraryRevision++ + a.mu.Unlock() + _ = a.persistLibrary() + a.mu.Lock() + close(a.libraryLoading) + a.libraryLoading = nil + a.libraryCancel = nil + a.libraryLoadingPhase = "" + a.libraryRevision++ + count := len(a.dramas) + a.mu.Unlock() + fmt.Printf("本地剧库:%d 部,本次新增 %d 部\n", count, count-previousCount) + }() +} + +func (a *UIApp) persistLibraryIfDirty() error { + a.mu.Lock() + dirty := a.libraryDirty + a.mu.Unlock() + if !dirty { + return nil + } + return a.persistLibrary() +} + +func (a *UIApp) markLibrarySourcesFailedLocked(ctx context.Context, source string, err error) { + if a.librarySources == nil { + a.librarySources = map[string]librarySourceState{} + } + if matchesSourceFilter(sourceHongguo, source) { + state := a.librarySources[sourceHongguo] + if state.Status == "loading" { + state.Status = "failed" + if state.Count > 0 { + state.Status = "partial" + } + state.Error = a.redactError(err) + state.Phase = "" + a.librarySources[sourceHongguo] = state + } + } + a.libraryError = librarySourceErrors(a.librarySources) +} + +func (a *UIApp) markLibrarySourcesCanceledLocked(ctx context.Context, source string) { + if a.librarySources == nil { + a.librarySources = map[string]librarySourceState{} + } + for _, provider := range []string{sourceHongguo} { + if !matchesSourceFilter(provider, source) || !sourceAllowed(ctx, provider) { + continue + } + state := a.librarySources[provider] + if state.Status == "loading" { + state.Status = "canceled" + state.Error = "" + state.Phase = "" + a.librarySources[provider] = state + } + } + a.libraryError = librarySourceErrors(a.librarySources) +} + +func (a *UIApp) acceptLibraryProgress(source string, items []Drama, loadErr error, done bool) error { + if canonicalProviderSource(source) != sourceHongguo { + return nil + } + source = sourceHongguo + a.mu.Lock() + items = onlyHongguoDramas(items) + a.normalizeDramaCovers(items) + newDramas := a.newSortMetadataDramasLocked(items) + if source == sourceHongguo { + a.libraryApp = a.downloader.hongguoCatalogSnapshot() + } + if done { + var mergeErr error + if loadErr != nil { + mergeErr = &libraryLoadError{failures: map[string]error{source: loadErr}} + } + a.dramas = mergeSourceDramas(a.dramas, items, mergeErr, source) + } else { + positions := make(map[string]int, len(a.dramas)) + for index, drama := range a.dramas { + positions[drama.ID] = index + } + for _, drama := range items { + if drama.ID == "" { + continue + } + if position, found := positions[drama.ID]; found { + a.dramas[position] = mergeDramaMetadata(drama, a.dramas[position]) + } else { + positions[drama.ID] = len(a.dramas) + a.dramas = append(a.dramas, drama) + } + } + } + a.enqueueSortMetadataLocked(newDramas, false) + state := a.librarySources[source] + state.Count = 0 + for _, drama := range a.dramas { + if dramaProvider(drama) == source { + state.Count++ + } + } + if len(items) > 0 { + a.loadedAt = time.Now() + state.UpdatedAt = a.loadedAt + } + if done { + state.Status = "ready" + if loadErr != nil { + state.Status = "failed" + if state.Count > 0 { + state.Status = "partial" + } + state.Error = a.redactError(loadErr) + } + state.Phase = "" + } + a.librarySources[source] = state + a.libraryError = librarySourceErrors(a.librarySources) + a.libraryDirty = true + a.libraryRevision++ + persist := done || len(items) > 0 && source == sourceHongguo || time.Since(a.libraryLastSave) >= 3*time.Second + if persist { + a.libraryLastSave = time.Now() + } + a.mu.Unlock() + if persist { + return a.persistLibrary() + } + return nil +} + +func librarySourceErrors(sources map[string]librarySourceState) string { + var messages []string + for source, state := range sources { + if state.Error != "" { + messages = append(messages, fmt.Sprintf("%s 获取失败: %s", source, state.Error)) + } + } + sort.Strings(messages) + return strings.Join(messages, "\n") +} + +func cloneLibrarySources(sources map[string]librarySourceState) map[string]librarySourceState { + copy := make(map[string]librarySourceState, len(sources)) + for source, state := range sources { + copy[source] = state + } + return copy +} + +func (a *UIApp) persistLibrary() error { + a.libraryWriteMu.Lock() + defer a.libraryWriteMu.Unlock() + a.mu.Lock() + cache := libraryCache{Dramas: append([]Drama{}, a.dramas...), LoadedAt: a.loadedAt, LastError: librarySourceErrors(a.librarySources), Sources: cloneLibrarySources(a.librarySources), HongguoApp: cloneHongguoCatalogState(a.libraryApp)} + revision := a.libraryRevision + a.mu.Unlock() + err := writeLibraryCache(a.cfg.dataDirectory(), cache) + a.mu.Lock() + defer a.mu.Unlock() + if err != nil { + a.libraryError = strings.TrimSpace(cache.LastError + "\n剧库缓存保存失败: " + a.redactError(err)) + a.libraryRevision++ + return err + } + a.librarySaved = true + if revision == a.libraryRevision { + a.libraryDirty = false + } + return nil +} + +func (a *UIApp) librarySnapshotLocked(revision uint64) map[string]any { + remaining := sortMetadataRemaining(a.dramas) + var unqueued []Drama + for _, drama := range a.dramas { + if !a.metadataPending[drama.ID] { + unqueued = append(unqueued, drama) + } + } + more := map[string]bool{} + for source, count := range sortMetadataRemaining(unqueued) { + more[source] = count > 0 + } + more["hongguo"] = more["hongguo"] || hongguoCatalogHasMore(a.libraryApp) + more[""] = more[""] || more["hongguo"] + response := map[string]any{ + "revision": a.libraryRevision, "loadedAt": a.loadedAt, "loading": a.libraryLoading != nil, + "cached": a.librarySaved && !a.libraryDirty, "error": a.libraryError, + "sources": cloneLibrarySources(a.librarySources), "total": len(a.dramas), + "hasMore": more["hongguo"], "hasMoreBySource": more, "metadataRemaining": remaining, + "loadingMore": a.libraryMore && a.libraryLoading != nil, "loadingSource": a.libraryLoadingSource, "loadingPhase": a.libraryLoadingPhase, "metadata": a.libraryMetadata, + } + if revision != 0 && revision == a.libraryRevision { + response["unchanged"] = true + } else { + response["data"] = append([]Drama{}, a.dramas...) + } + return response +} diff --git a/internal/app/library_update_test.go b/internal/app/library_update_test.go new file mode 100644 index 0000000..6e16c12 --- /dev/null +++ b/internal/app/library_update_test.go @@ -0,0 +1,322 @@ +package app + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestUnifiedUpdateChecksNewContentAndContinuesCatalog(t *testing.T) { + for _, test := range []struct { + name string + pages int + savedOffset int + savedEnd bool + knownOffsets []int + endAt int + wantOffsets []int + wantSaved int + wantEnd bool + resumeSession bool + }{ + {name: "first import runs one batch", pages: 2, endAt: -1, wantOffsets: []int{0, 18}, wantSaved: 36}, + {name: "known head resumes saved tail", pages: 2, savedOffset: 90, knownOffsets: []int{0}, endAt: -1, wantOffsets: []int{0, 90, 108}, wantSaved: 126, resumeSession: true}, + {name: "new head stops at overlap", pages: 4, savedOffset: 90, knownOffsets: []int{18}, endAt: -1, wantOffsets: []int{0, 18, 90, 108, 126, 144}, wantSaved: 162, resumeSession: true}, + {name: "large update continues next unread page", pages: 4, savedOffset: 180, endAt: -1, wantOffsets: []int{0, 18, 36, 54, 72, 90, 108}, wantSaved: 126}, + {name: "exhausted catalog still checks new content", pages: 4, savedOffset: 180, savedEnd: true, knownOffsets: []int{18}, endAt: -1, wantOffsets: []int{0, 18}, wantSaved: 180, wantEnd: true}, + {name: "continuation reaches end", pages: 2, savedOffset: 90, knownOffsets: []int{0}, endAt: 90, wantOffsets: []int{0, 90}, wantSaved: 108, wantEnd: true, resumeSession: true}, + } { + t.Run(test.name, func(t *testing.T) { + calls := map[string][]int{} + var known []Drama + for index := range hongguoAppGenres { + for _, offset := range test.knownOffsets { + known = append(known, Drama{ID: fmt.Sprintf("hongguo:%d", 790000+index*1000+offset), Source: sourceHongguo}) + } + } + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + var payload struct { + Offset int `json:"offset"` + Session string `json:"session_id"` + Select struct { + Genre []string `json:"genre"` + } `json:"select_items"` + } + if request.URL.Path != "/reading/distribution/category/landpage/v/" || json.NewDecoder(request.Body).Decode(&payload) != nil || len(payload.Select.Genre) != 1 { + t.Errorf("unexpected catalog request: %s", request.URL.Path) + return rankingHTTPResponse(request, 400, ""), nil + } + genre := payload.Select.Genre[0] + index := -1 + for position, candidate := range hongguoAppGenres { + if candidate.key == genre { + index = position + } + } + if index < 0 { + t.Errorf("unexpected genre: %s", genre) + return rankingHTTPResponse(request, 400, ""), nil + } + calls[genre] = append(calls[genre], payload.Offset) + if payload.Offset == 0 && payload.Session != "" { + t.Error("head check reused a tail session") + } + if test.resumeSession && payload.Offset == test.savedOffset && payload.Session != "saved-"+genre { + t.Error("continuation lost its saved session") + } + body, _ := json.Marshal(map[string]any{"data": map[string]any{ + "next_offset": payload.Offset + 18, "session_id": "fixture-" + genre, "has_more": payload.Offset != test.endAt, + "video_data": []any{map[string]any{ + "series_id": fmt.Sprint(790000 + index*1000 + payload.Offset), "series_title": "文字目录样本", + "first_visible_time": "1773662280", "hot_score": "20", "series_play_cnt": "0", + }}, + }}) + return rankingHTTPResponse(request, 200, string(body)), nil + }) + d.cfg.MaxPagesPerSort = test.pages + if test.savedOffset > 0 { + client := d.hongguoClient() + for _, genre := range hongguoAppGenres { + client.state.Feeds[genre.key] = hongguoCatalogCursor{ + Offset: test.savedOffset, SessionID: "saved-" + genre.key, Initialized: true, Exhausted: test.savedEnd, UpdatedAt: time.Now(), + } + } + } + ctx := withKnownHongguoDramas(context.WithValue(context.Background(), libraryUpdateKey{}, true), known) + items, err := d.fetchHongguoAppCatalog(ctx) + if err != nil || len(items) != len(test.wantOffsets)*len(hongguoAppGenres) { + t.Fatalf("catalog result: %d items, %v", len(items), err) + } + for _, genre := range hongguoAppGenres { + if !reflect.DeepEqual(calls[genre.key], test.wantOffsets) { + t.Errorf("%s requested %v, want %v", genre.key, calls[genre.key], test.wantOffsets) + } + cursor := d.hongguoCatalogSnapshot().Feeds[genre.key] + if !cursor.Initialized || cursor.Offset != test.wantSaved || cursor.Exhausted != test.wantEnd { + t.Errorf("wrong checkpoint for %s: %+v", genre.key, cursor) + } + } + }) + } +} + +func TestUnifiedUpdatePreservesHistoryAcrossRestart(t *testing.T) { + calls := map[string][]int{} + transport := func(request *http.Request) (*http.Response, error) { + var payload struct { + Offset int `json:"offset"` + Scene string `json:"req_scene"` + } + if request.URL.Path != "/reading/distribution/category/landpage/v/" || json.NewDecoder(request.Body).Decode(&payload) != nil { + t.Errorf("unexpected request: %s", request.URL.Path) + return rankingHTTPResponse(request, 400, ""), nil + } + calls[payload.Scene] = append(calls[payload.Scene], payload.Offset) + index := 0 + for position, genre := range hongguoAppGenres { + if genre.scene == payload.Scene { + index = position + } + } + body := fmt.Sprintf(`{"data":{"next_offset":%d,"has_more":%t,"session_id":"saved-session","video_data":[{"series_id":"%d","series_title":"文字样本","first_visible_time":"1773662280","hot_score":"20","series_play_cnt":"0"}]}}`, payload.Offset+18, payload.Offset == 0, 790000+index*1000+payload.Offset) + return rankingHTTPResponse(request, 200, body), nil + } + directory := t.TempDir() + for run, wantCount := range []int{3, 6, 6} { + d := rankingTestDownloader(t, transport) + d.cfg.dataDir, d.cfg.MaxPagesPerSort = directory, 1 + items, err := d.RefreshDramas(context.WithValue(context.Background(), libraryUpdateKey{}, true), sourceHongguo) + if err != nil || len(items) != wantCount { + t.Fatalf("run %d lost history: %d items, %v", run, len(items), err) + } + cache, err := readLibraryCache(directory) + if err != nil || len(cache.Dramas) != wantCount || hongguoCatalogHasMore(cache.HongguoApp) != (run == 0) { + t.Fatalf("run %d did not persist data and cursor together: %v", run, err) + } + for _, drama := range cache.Dramas { + if drama.OnlineDate != "2026-03-16" || drama.Heat != "20" || drama.Views != "0" { + t.Fatal("update lost available sorting fields") + } + } + } + for scene, offsets := range calls { + if !reflect.DeepEqual(offsets, []int{0, 0, 18, 0}) { + t.Errorf("%s skipped or repeated a continuation: %v", scene, offsets) + } + } +} + +func TestUnifiedUpdateReturnsImmediatelyAndDeduplicatesWork(t *testing.T) { + catalogStarted, catalogRelease := make(chan struct{}), make(chan struct{}) + detailStarted, detailRelease := make(chan string, 1), make(chan struct{}) + releaseCatalog := sync.OnceFunc(func() { close(catalogRelease) }) + releaseDetail := sync.OnceFunc(func() { close(detailRelease) }) + var catalogOnce sync.Once + var catalogCalls atomic.Int32 + var detailMu sync.Mutex + var detailIDs []string + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + switch request.URL.Path { + case "/reading/distribution/category/landpage/v/": + catalogCalls.Add(1) + catalogOnce.Do(func() { close(catalogStarted) }) + select { + case <-catalogRelease: + case <-request.Context().Done(): + return nil, request.Context().Err() + } + return rankingHTTPResponse(request, 200, `{"data":{"next_offset":18,"has_more":false,"video_data":[{"series_id":"790000","series_title":"已完成的目录条目","first_visible_time":"1773662280","hot_score":"20","series_play_cnt":"0"}]}}`), nil + case "/detail": + if request.Context().Value(backgroundCatalogKey{}) != true { + t.Error("historical metadata did not use background priority") + } + id := request.URL.Query().Get("series_id") + detailMu.Lock() + detailIDs = append(detailIDs, id) + if len(detailIDs) == 1 { + detailStarted <- id + } + detailMu.Unlock() + select { + case <-detailRelease: + case <-request.Context().Done(): + return nil, request.Context().Err() + } + return rankingHTTPResponse(request, 200, sortDetailFixture(id, "1773662280")), nil + default: + t.Errorf("unexpected resource: %s", request.URL.Path) + return rankingHTTPResponse(request, 404, ""), nil + } + }) + d.cfg.MaxPagesPerSort = 1 + for _, genre := range hongguoAppGenres { + d.hongguoClient().state.Feeds[genre.key] = hongguoCatalogCursor{Initialized: true, Exhausted: true, Offset: 18} + } + app := &UIApp{downloader: d, cfg: d.cfg, libraryAttempted: true, libraryApp: d.hongguoCatalogSnapshot(), librarySources: map[string]librarySourceState{}} + app.dramas = append(app.dramas, Drama{ID: "hongguo:790000", Source: sourceHongguo, Title: "已完成的目录条目", OnlineDate: "2026-03-16", Heat: "20", Views: "0", Cover: coverAddressFixture, SortMetadata: &sortMetadataState{Version: sortMetadataVersion}}) + for index := 0; index < 60; index++ { + app.dramas = append(app.dramas, Drama{ID: fmt.Sprintf("hongguo:%d", 800000+index), Source: sourceHongguo, Title: "历史文字样本", Heat: "20", Views: "0", Cover: coverAddressFixture}) + } + t.Cleanup(func() { + releaseCatalog() + releaseDetail() + app.mu.Lock() + done, cancel := app.libraryLoading, app.libraryCancel + app.mu.Unlock() + if cancel != nil { + cancel() + } + if done != nil { + select { + case <-done: + case <-time.After(3 * time.Second): + t.Error("catalog worker did not stop") + } + } + app.stopSortMetadata() + }) + type snapshot struct { + Accepted bool `json:"updateAccepted"` + Loading bool `json:"loading"` + Data []Drama `json:"data"` + Metadata libraryMetadataProgress `json:"metadata"` + } + read := func(path string) snapshot { + t.Helper() + done := make(chan *httptest.ResponseRecorder, 1) + go func() { + writer := httptest.NewRecorder() + app.handleDramas(writer, httptest.NewRequest("GET", path, nil)) + done <- writer + }() + select { + case writer := <-done: + var result snapshot + if writer.Code != 200 || json.Unmarshal(writer.Body.Bytes(), &result) != nil { + t.Fatalf("invalid snapshot: %d %s", writer.Code, writer.Body.String()) + } + return result + case <-time.After(2 * time.Second): + t.Fatal("library response waited for a stalled upstream request") + return snapshot{} + } + } + if result := read("/api/ui/dramas"); result.Loading || result.Metadata.Total != 0 || catalogCalls.Load() != 0 { + t.Fatal("ordinary cache read scheduled remote work") + } + path := "/api/ui/dramas?update=1&source=hongguo&priority=hongguo:800059" + result := read(path) + if !result.Accepted || !result.Loading || result.Metadata.Total != 40 || len(result.Data) != 61 { + t.Fatalf("update did not immediately return cache and bounded progress: %+v", result.Metadata) + } + select { + case <-catalogStarted: + case <-time.After(2 * time.Second): + t.Fatal("update did not start the main catalog") + } + for _, request := range []string{path, path, "/api/ui/dramas?revision=1"} { + if result := read(request); result.Accepted || !result.Loading || result.Metadata.Total != 40 { + t.Fatal("repeated update or polling duplicated work") + } + } + select { + case <-detailStarted: + t.Fatal("metadata did not yield to the catalog") + default: + } + app.mu.Lock() + catalogDone := app.libraryLoading + app.mu.Unlock() + releaseCatalog() + select { + case <-catalogDone: + case <-time.After(2 * time.Second): + t.Fatal("catalog waited for optional metadata") + } + select { + case id := <-detailStarted: + if id != "800059" { + t.Fatalf("current filtered result was not prioritized: %s", id) + } + case <-time.After(2 * time.Second): + t.Fatal("historical metadata was not scheduled") + } + if result := read("/api/ui/dramas"); result.Loading || !result.Metadata.Running || result.Metadata.Checked != 0 || catalogCalls.Load() != 3 { + t.Fatal("optional metadata blocked the main response or catalog work was duplicated") + } + releaseDetail() + waitSortMetadataQueue(t, app) + result = read("/api/ui/dramas") + if result.Metadata.Checked != 40 || result.Metadata.Failed != 0 || result.Metadata.Running { + t.Fatalf("unexpected completion: %+v", result.Metadata) + } + untouched := 0 + for _, drama := range result.Data { + if strings.HasPrefix(drama.ID, "hongguo:800") && drama.SortMetadata == nil { + untouched++ + } + } + if untouched != 20 { + t.Fatalf("update migrated more than one historical batch: %d untouched", untouched) + } + detailMu.Lock() + defer detailMu.Unlock() + seen := map[string]bool{} + for _, id := range detailIDs { + if seen[id] { + t.Fatalf("duplicate historical request: %s", id) + } + seen[id] = true + } + if len(seen) != 40 { + t.Fatalf("unexpected detail request count: %d", len(seen)) + } +} diff --git a/internal/app/media_crypto.go b/internal/app/media_crypto.go new file mode 100644 index 0000000..54e60b4 --- /dev/null +++ b/internal/app/media_crypto.go @@ -0,0 +1,21 @@ +package app + +import ( + "errors" +) + +func pkcs7Unpad(src []byte, blockSize int) ([]byte, error) { + if len(src) == 0 || len(src)%blockSize != 0 { + return nil, errors.New("invalid pkcs7 length") + } + pad := int(src[len(src)-1]) + if pad == 0 || pad > blockSize || pad > len(src) { + return nil, errors.New("invalid pkcs7 padding") + } + for _, v := range src[len(src)-pad:] { + if int(v) != pad { + return nil, errors.New("invalid pkcs7 padding bytes") + } + } + return src[:len(src)-pad], nil +}