diff --git a/internal/app/ffmpeg_install_test.go b/internal/app/ffmpeg_install_test.go new file mode 100644 index 0000000..ef4140e --- /dev/null +++ b/internal/app/ffmpeg_install_test.go @@ -0,0 +1,309 @@ +package app + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestMain(m *testing.M) { + if os.Getenv("JUKU_TEST_FFMPEG_HELPER") == "1" { + ffmpegTestHelper() + return + } + os.Exit(m.Run()) +} + +func ffmpegTestHelper() { + mode, _ := os.ReadFile(os.Args[0] + ".mode") + if log, err := os.OpenFile(os.Args[0]+".calls", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600); err == nil { + fmt.Fprintln(log, strings.Join(os.Args[1:], " ")) + _ = log.Close() + } + if string(mode) == "hang" { + time.Sleep(time.Minute) + } + if string(mode) == "slow" { + time.Sleep(100 * time.Millisecond) + } + if strings.Contains(strings.Join(os.Args[1:], " "), "-encoders") { + fmt.Println(" A..... aac AAC") + if string(mode) != "missing" { + fmt.Println(" V..... libx264 H.264") + } + return + } + if string(mode) == "preset" { + fmt.Fprintln(os.Stderr, "Unrecognized option 'preset'.\r\nError splitting the argument list: Option not found") + os.Exit(8) + } + fmt.Print("synthetic MP4 output") +} + +func ffmpegTestEnvironment(t *testing.T) string { + t.Helper() + previous, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + if err := os.Chdir(root); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(previous) }) + t.Setenv("PATH", "") + t.Setenv("JUKU_TEST_FFMPEG_HELPER", "1") + return root +} + +func ffmpegTestProgram(t *testing.T, directory, mode string) string { + t.Helper() + if err := os.MkdirAll(directory, 0755); err != nil { + t.Fatal(err) + } + program, err := os.Executable() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(directory, ffmpegExecutableName()) + if err := os.Link(program, path); err != nil { + body, err := os.ReadFile(program) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, body, 0755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(path+".mode", []byte(mode), 0600); err != nil { + t.Fatal(err) + } + return path +} + +func ffmpegTestInstaller(t *testing.T, configured string) *ffmpegInstaller { + t.Helper() + return &ffmpegInstaller{ + configured: configured, name: ffmpegExecutableName(), directory: ffmpegManagedDirectory(), + client: &http.Client{Transport: rankingTransport(func(*http.Request) (*http.Response, error) { + t.Error("unexpected FFmpeg download") + return nil, errors.New("network forbidden") + })}, + } +} + +func TestFFmpegPATHPrecedesPortableAndKeepsAutomaticSetting(t *testing.T) { + root := ffmpegTestEnvironment(t) + ffmpegTestProgram(t, filepath.Join(root, "bin"), "preset") + ffmpegTestProgram(t, root, "preset") + full := ffmpegTestProgram(t, filepath.Join(root, "系统 PATH with spaces"), "good") + t.Setenv("PATH", filepath.Dir(full)) + d := NewDownloader(Config{FFmpeg: "ffmpeg", OutputDir: filepath.Join(root, "out"), dataDir: filepath.Join(root, "data"), settingsLoaded: true}) + if d.cfg.FFmpeg != "ffmpeg" { + t.Fatal("automatic discovery was frozen as an explicit portable path", d.cfg.FFmpeg) + } + if d.ffmpegInstallation().snapshot().Status == "ready" { + t.Fatal("unverified executable advertised as ready") + } + path, err := d.ensureFFmpeg(context.Background()) + if err != nil || path != full { + t.Fatal("PATH must take precedence over current-directory and bundled programs", path, err) + } + if !strings.Contains(d.ffmpegInstallation().snapshot().Detail, full) { + t.Fatal("the selected executable must be visible in status") + } +} + +func TestFFmpegSkipsIncompatiblePATHCandidates(t *testing.T) { + root := ffmpegTestEnvironment(t) + first := ffmpegTestProgram(t, filepath.Join(root, "minimal"), "missing") + second := ffmpegTestProgram(t, filepath.Join(root, "advertised-only"), "preset") + full := ffmpegTestProgram(t, filepath.Join(root, "full"), "good") + t.Setenv("PATH", strings.Join([]string{filepath.Dir(first), filepath.Dir(second), filepath.Dir(first), filepath.Dir(full)}, string(os.PathListSeparator))) + installer := ffmpegTestInstaller(t, "ffmpeg") + var events []diagnosticEvent + installer.record = func(event diagnosticEvent) { events = append(events, event) } + path, err := installer.ensure(context.Background()) + if err != nil || path != full { + t.Fatal("a PATH hit without working encoders must not mask a usable fallback", path, err) + } + if len(events) != 3 || events[0].Event != "ffmpeg.rejected" || !strings.Contains(events[1].Message, "preset") || events[2].Event != "ffmpeg.ready" || !strings.Contains(events[2].Message, full) { + t.Fatal("selection diagnostics must explain rejected candidates and the final path", events) + } + if candidates := ffmpegCandidates("ffmpeg", installer.directory); len(candidates) != 3 { + t.Fatal("duplicate PATH entries were not deduplicated", candidates) + } +} + +func TestFFmpegExplicitPathsRemainAuthoritative(t *testing.T) { + root := ffmpegTestEnvironment(t) + full := ffmpegTestProgram(t, filepath.Join(root, "full"), "good") + bad := ffmpegTestProgram(t, filepath.Join(root, "explicit"), "preset") + t.Setenv("PATH", filepath.Dir(full)) + installer := ffmpegTestInstaller(t, bad) + path, err := installer.ensure(context.Background()) + if err == nil || path != "" || !strings.Contains(err.Error(), "preset") || !strings.Contains(err.Error(), bad) { + t.Fatal("an invalid explicit path must report its failure without switching programs", path, err) + } + if installer.snapshot().Status != "failed" { + t.Fatal("invalid explicit program was marked ready") + } + if err := os.WriteFile(bad+".mode", []byte("good"), 0600); err != nil { + t.Fatal(err) + } + installer.retry() + path, err = installer.ensure(context.Background()) + if err != nil || path != bad { + t.Fatal("retry must probe a corrected explicit program", path, err) + } + installer = ffmpegTestInstaller(t, " \""+full+"\" ") + path, err = installer.ensure(context.Background()) + if err != nil || path != full { + t.Fatal("quoted paths with spaces were not resolved", path, err) + } + installer = ffmpegTestInstaller(t, filepath.Join(root, "missing", "ffmpeg.exe")) + if _, err := installer.ensure(context.Background()); err == nil || !strings.Contains(err.Error(), "找不到指定") { + t.Fatal("missing explicit path must not silently download", err) + } +} + +func TestFFmpegSharedProbeIsCachedAndCancelable(t *testing.T) { + root := ffmpegTestEnvironment(t) + program := ffmpegTestProgram(t, filepath.Join(root, "slow"), "slow") + installer := ffmpegTestInstaller(t, program) + var wait sync.WaitGroup + for index := 0; index < 8; index++ { + wait.Add(1) + go func() { + defer wait.Done() + if path, err := installer.ensure(context.Background()); err != nil || path != program { + t.Error(path, err) + } + }() + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := installer.ensure(ctx); !errors.Is(err, context.Canceled) { + t.Fatal("canceled waiter did not exit", err) + } + wait.Wait() + if _, err := installer.ensure(context.Background()); err != nil { + t.Fatal(err) + } + log, err := os.ReadFile(program + ".calls") + if err != nil || len(strings.Split(strings.TrimSpace(string(log)), "\n")) != 2 { + t.Fatal("concurrent users must share one encoder query and one real probe", string(log), err) + } + hanging := ffmpegTestProgram(t, filepath.Join(root, "hang"), "hang") + ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + start := time.Now() + if err := validateFFmpeg(ctx, hanging); err == nil || time.Since(start) > 3*time.Second { + t.Fatal("capability probe must honor cancellation", err, time.Since(start)) + } +} + +func ffmpegTestPackage(t *testing.T, installer *ffmpegInstaller) *atomic.Int32 { + t.Helper() + program, err := os.Executable() + if err != nil { + t.Fatal(err) + } + binary, err := os.ReadFile(program) + if err != nil { + t.Fatal(err) + } + var archive bytes.Buffer + writer, _ := gzip.NewWriterLevel(&archive, gzip.BestSpeed) + _, _ = writer.Write(binary) + if err := writer.Close(); err != nil { + t.Fatal(err) + } + digest := func(body []byte) string { value := sha256.Sum256(body); return hex.EncodeToString(value[:]) } + document := []byte("synthetic test package documentation") + installer.pack = ffmpegPackage{platform: "fixture", archiveSize: int64(archive.Len()), binarySize: int64(len(binary)), archiveSHA256: digest(archive.Bytes()), binarySHA256: digest(binary), licenseSHA256: digest(document), readmeSHA256: digest(document)} + var requests atomic.Int32 + installer.client = &http.Client{Transport: rankingTransport(func(request *http.Request) (*http.Response, error) { + requests.Add(1) + body := document + if strings.HasSuffix(request.URL.Path, ".gz") { + body = archive.Bytes() + } + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(body)), ContentLength: int64(len(body)), Request: request}, nil + })} + return &requests +} + +func TestFFmpegAutomaticFallbackPreservesExistingPortableProgram(t *testing.T) { + root := ffmpegTestEnvironment(t) + old := ffmpegTestProgram(t, filepath.Join(root, "bin", runtime.GOOS+"-"+runtime.GOARCH), "missing") + info, err := os.Stat(old) + if err != nil { + t.Fatal(err) + } + installer := ffmpegTestInstaller(t, "ffmpeg") + requests := ffmpegTestPackage(t, installer) + path, err := installer.ensure(context.Background()) + if err != nil || path != filepath.Join(installer.directory, installer.name) || path == old { + t.Fatal("verified auto-download must be selected after rejecting the old portable program", path, err) + } + if requests.Load() != 3 { + t.Fatal("archive, license and build details must all be fetched", requests.Load()) + } + after, err := os.Stat(old) + if err != nil || !os.SameFile(info, after) { + t.Fatal("existing portable program was replaced", err) + } + restarted := ffmpegTestInstaller(t, "ffmpeg") + if next, err := restarted.ensure(context.Background()); err != nil || next != path { + t.Fatal("restart must reuse the verified installation without downloading", next, err) + } +} + +func TestFFmpegInstallationRejectsCorruptionAndInvalidDestination(t *testing.T) { + root := ffmpegTestEnvironment(t) + installer := ffmpegTestInstaller(t, "ffmpeg") + ffmpegTestPackage(t, installer) + installer.pack.archiveSHA256 = strings.Repeat("0", 64) + if _, err := installer.install(context.Background()); err == nil || !strings.Contains(err.Error(), "SHA-256") { + t.Fatal("corrupt download was accepted", err) + } + if _, err := os.Stat(installer.directory); !errors.Is(err, os.ErrNotExist) { + t.Fatal("a corrupt download must not create a usable installation", err) + } + ffmpegTestPackage(t, installer) + bad := ffmpegTestProgram(t, installer.directory, "preset") + if path, err := installer.install(context.Background()); err == nil || path != "" { + t.Fatal("an existing invalid destination must not turn an install failure into success", path, err) + } + if mode, err := os.ReadFile(bad + ".mode"); err != nil || string(mode) != "preset" { + t.Fatal("existing installation contents were changed", root, err) + } +} + +func TestFFmpegLocalPlaybackProbe(t *testing.T) { + program, err := exec.LookPath("ffmpeg") + if err != nil { + t.Skip("local FFmpeg unavailable") + } + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + defer cancel() + if err := validateFFmpeg(ctx, program); err != nil { + t.Fatal(err) + } +} diff --git a/internal/app/ffmpeg_media.go b/internal/app/ffmpeg_media.go new file mode 100644 index 0000000..3915258 --- /dev/null +++ b/internal/app/ffmpeg_media.go @@ -0,0 +1,22 @@ +package app + +import ( + "context" + "os" + "os/exec" + "strings" +) + +func ffmpegMediaCommand(ctx context.Context, executable string, args ...string) *exec.Cmd { + command := exec.CommandContext(ctx, executable, args...) + command.Env = make([]string, 0, len(os.Environ())) + for _, entry := range os.Environ() { + key, _, _ := strings.Cut(entry, "=") + switch strings.ToLower(key) { + case "http_proxy", "https_proxy", "all_proxy": + continue + } + command.Env = append(command.Env, entry) + } + return command +} diff --git a/internal/app/ffmpeg_media_test.go b/internal/app/ffmpeg_media_test.go new file mode 100644 index 0000000..1c42b3b --- /dev/null +++ b/internal/app/ffmpeg_media_test.go @@ -0,0 +1,91 @@ +package app + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +func TestFFmpegLoopbackKeepsApplicationProxy(t *testing.T) { + fixture, session := nativePlaybackFixture(t, "3", "640x360") + video, err := os.ReadFile(session.tasks[0].OutPath) + if err != nil { + t.Fatal(err) + } + var inheritedRequests, mediaRequests atomic.Int32 + inheritedProxy := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + inheritedRequests.Add(1) + http.Error(writer, "loopback must not reach the inherited proxy", http.StatusBadGateway) + })) + t.Cleanup(inheritedProxy.Close) + applicationProxy := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.URL.Host != "media.example.invalid" || request.URL.Path != "/fixture.mp4" { + t.Error("application proxy received an unrelated request") + http.NotFound(writer, request) + return + } + mediaRequests.Add(1) + http.ServeContent(writer, request, "fixture.mp4", time.Time{}, bytes.NewReader(video)) + })) + t.Cleanup(applicationProxy.Close) + for _, key := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"} { + t.Setenv(key, inheritedProxy.URL) + } + for _, key := range []string{"no_proxy", "NO_PROXY"} { + t.Setenv(key, "") + } + for _, mode := range []string{"MSE", "MSE remux", "native HLS", "download"} { + t.Run(mode, func(t *testing.T) { + app, _ := prefetchFixtureApp(t) + root := t.TempDir() + app.downloader = NewDownloader(Config{FFmpeg: fixture.downloader.cfg.FFmpeg, ProxyURL: applicationProxy.URL, + OutputDir: filepath.Join(root, "out"), dataDir: filepath.Join(root, "data"), settingsLoaded: true, Retries: 1, SkipBytes: 1}) + app.cfg = app.downloader.cfg + task := Task{DramaID: "hongguo:7000000000000000001", DramaTitle: "合成代理样本", Index: 1, + OutPath: filepath.Join(root, "out", "episode.mp4"), Chapter: Chapter{ID: "hongguo:7000000000000000001:1", Source: sourceHongguo, + VideoURL: "http://media.example.invalid/fixture.mp4"}} + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + before := mediaRequests.Load() + switch mode { + case "MSE", "MSE remux": + ctx = context.WithValue(ctx, playbackRemuxKey{}, mode == "MSE remux") + writer := httptest.NewRecorder() + if err := app.streamPlayback(ctx, cancel, writer, task, "", 0, 1, func(float64) {}); err != nil { + t.Fatal("playback through the application proxy failed", err) + } + if !bytes.Contains(writer.Body.Bytes(), []byte("moof")) { + t.Fatal("playback did not return fragmented video") + } + case "native HLS": + cache := newPlaybackNative(app, ctx, task, "", 0, 0, false) + defer func() { + cache.Close() + cache.workers.Wait() + }() + cache.start() + body, err := cache.segment(ctx, 0) + if err != nil || len(body) < 188 || body[0] != 0x47 { + t.Fatal("native playback through the application proxy failed", err) + } + case "download": + if err := app.downloader.DownloadEpisode(ctx, task); err != nil { + t.Fatal("download through the application proxy failed", err) + } + info, err := os.Stat(task.OutPath) + if err != nil || info.Size() < 100*1024 { + t.Fatal("download did not produce the fixture", err) + } + } + if inheritedRequests.Load() != 0 || mediaRequests.Load() <= before { + t.Fatalf("incorrect proxy route: inherited=%d media=%d", inheritedRequests.Load(), mediaRequests.Load()) + } + }) + } +} diff --git a/internal/app/ffmpeg_paths.go b/internal/app/ffmpeg_paths.go new file mode 100644 index 0000000..4dd3318 --- /dev/null +++ b/internal/app/ffmpeg_paths.go @@ -0,0 +1,80 @@ +package app + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +func normalizedFFmpegSetting(value string) string { + value = strings.TrimSpace(value) + if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { + value = strings.TrimSpace(value[1 : len(value)-1]) + } + return value +} + +func automaticFFmpeg(value string) bool { + value = normalizedFFmpegSetting(value) + if runtime.GOOS == "windows" { + value = strings.ToLower(value) + } + return value == "" || value == "ffmpeg" || value == "ffmpeg.exe" +} + +func ffmpegExecutableName() string { + if runtime.GOOS == "windows" { + return "ffmpeg.exe" + } + return "ffmpeg" +} + +func ffmpegManagedDirectory() string { + directory := filepath.Join("bin", runtime.GOOS+"-"+runtime.GOARCH, "verified-b6.1.1") + if absolute, err := filepath.Abs(directory); err == nil { + return absolute + } + return directory +} + +func ffmpegCandidates(configured, managedDirectory string) []string { + var candidates []string + seen := make(map[string]bool) + add := func(candidate string) { + path, err := exec.LookPath(candidate) + if err != nil { + return + } + path, err = filepath.Abs(path) + if err != nil { + return + } + key := path + if runtime.GOOS == "windows" { + key = strings.ToLower(key) + } + if !seen[key] { + seen[key] = true + candidates = append(candidates, path) + } + } + configured = normalizedFFmpegSetting(configured) + if !automaticFFmpeg(configured) { + add(configured) + return candidates + } + name := ffmpegExecutableName() + for _, directory := range filepath.SplitList(os.Getenv("PATH")) { + if filepath.IsAbs(directory) { + add(filepath.Join(directory, name)) + } + } + for _, path := range []string{filepath.Join("bin", name), name, filepath.Join(managedDirectory, name), filepath.Join("bin", runtime.GOOS+"-"+runtime.GOARCH, name)} { + if absolute, err := filepath.Abs(path); err == nil { + add(absolute) + } + } + return candidates +} diff --git a/internal/app/ffmpeg_probe_windows.go b/internal/app/ffmpeg_probe_windows.go new file mode 100644 index 0000000..3979d58 --- /dev/null +++ b/internal/app/ffmpeg_probe_windows.go @@ -0,0 +1,12 @@ +package app + +import ( + "os/exec" + "syscall" +) + +func init() { + configureFFmpegProbeProcess = func(command *exec.Cmd) { + command.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: 0x08000000} + } +} diff --git a/internal/app/following_store.go b/internal/app/following_store.go new file mode 100644 index 0000000..ed93990 --- /dev/null +++ b/internal/app/following_store.go @@ -0,0 +1,193 @@ +package app + +import ( + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "sort" + "sync" + "time" +) + +const followingLimit = 1000 + +var errFollowingLimit = errors.New("追剧清单已达到 1000 部,请先移除不再需要的内容") +var errFollowingUnknown = errors.New("剧库中没有这部剧,请先搜索或更新剧库") + +type followingEntry struct { + DramaID string `json:"dramaId"` + Source string `json:"source"` + Title string `json:"title"` + Category string `json:"category,omitempty"` + Saved bool `json:"saved"` + Completed bool `json:"completed"` + KnownEpisodes int `json:"knownEpisodes"` + AddedAt time.Time `json:"addedAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type followingFile struct { + Version int `json:"version"` + Entries []followingEntry `json:"entries"` +} + +type followingStore struct { + mu sync.Mutex + path string + entries map[string]followingEntry + loadErr error +} + +func newFollowingStore(directory string) *followingStore { + store := &followingStore{path: filepath.Join(directory, "following.json"), entries: map[string]followingEntry{}} + file, err := os.Open(store.path) + if errors.Is(err, os.ErrNotExist) { + return store + } + if err != nil { + store.loadErr = err + return store + } + defer file.Close() + var saved followingFile + decoder := json.NewDecoder(io.LimitReader(file, 8<<20)) + if err = decoder.Decode(&saved); err == nil { + if decoder.Decode(new(any)) != io.EOF || saved.Version != 1 || len(saved.Entries) > followingLimit { + err = errors.New("追剧清单格式无效,原文件已保留") + } + } + if err != nil { + store.loadErr = err + return store + } + for _, entry := range saved.Entries { + entry, valid := normalizeFollowingEntry(entry) + if !valid { + continue + } + if old, exists := store.entries[entry.DramaID]; !exists || entry.UpdatedAt.After(old.UpdatedAt) { + store.entries[entry.DramaID] = entry + } + } + return store +} + +func followingEntries(entries map[string]followingEntry) []followingEntry { + result := make([]followingEntry, 0, len(entries)) + for _, entry := range entries { + result = append(result, entry) + } + sort.Slice(result, func(i, j int) bool { + if result[i].UpdatedAt.Equal(result[j].UpdatedAt) { + return result[i].DramaID < result[j].DramaID + } + return result[i].UpdatedAt.After(result[j].UpdatedAt) + }) + return result +} + +func normalizeFollowingEntry(entry followingEntry) (followingEntry, bool) { + id, source, valid := playbackHistoryIdentity(entry.DramaID) + if !valid || canonicalProviderSource(entry.Source) != source || len(entry.Title) > 4096 || len(entry.Category) > 512 || entry.KnownEpisodes < 0 || entry.KnownEpisodes > 10000 || entry.AddedAt.IsZero() || entry.UpdatedAt.IsZero() || !entry.Saved && !entry.Completed { + return followingEntry{}, false + } + entry.DramaID, entry.Source = id, source + return entry, true +} + +func (store *followingStore) list() ([]followingEntry, error) { + store.mu.Lock() + defer store.mu.Unlock() + return followingEntries(store.entries), store.loadErr +} + +func (store *followingStore) update(id string, change func(followingEntry, bool) (followingEntry, error)) (followingEntry, error) { + store.mu.Lock() + defer store.mu.Unlock() + if store.loadErr != nil { + return followingEntry{}, store.loadErr + } + previous, exists := store.entries[id] + entry, err := change(previous, exists) + if err != nil { + return followingEntry{}, err + } + if !exists && (entry.Saved || entry.Completed) && len(store.entries) >= followingLimit { + return followingEntry{}, errFollowingLimit + } + next := make(map[string]followingEntry, len(store.entries)+1) + for key, value := range store.entries { + next[key] = value + } + if entry.Saved || entry.Completed { + next[id] = entry + } else { + delete(next, id) + } + if err = store.write(followingFile{Version: 1, Entries: followingEntries(next)}); err != nil { + return followingEntry{}, err + } + store.entries = next + return entry, nil +} + +func (store *followingStore) merge(entries []followingEntry) (int, int, error) { + store.mu.Lock() + defer store.mu.Unlock() + if store.loadErr != nil { + return 0, 0, store.loadErr + } + next := make(map[string]followingEntry, len(store.entries)+len(entries)) + for id, entry := range store.entries { + next[id] = entry + } + imported := 0 + skipped := 0 + for _, entry := range entries { + entry, valid := normalizeFollowingEntry(entry) + if !valid { + continue + } + current, exists := next[entry.DramaID] + if exists && !entry.UpdatedAt.After(current.UpdatedAt) { + continue + } + if !exists && len(next) >= followingLimit { + skipped++ + continue + } + next[entry.DramaID] = entry + imported++ + } + if imported == 0 { + return 0, skipped, nil + } + if err := store.write(followingFile{Version: 1, Entries: followingEntries(next)}); err != nil { + return 0, skipped, err + } + store.entries = next + return imported, skipped, nil +} + +func (store *followingStore) write(data followingFile) error { + if err := os.MkdirAll(filepath.Dir(store.path), 0o700); err != nil { + return err + } + file, err := os.CreateTemp(filepath.Dir(store.path), ".following-*.json") + if err != nil { + return err + } + defer os.Remove(file.Name()) + if err = json.NewEncoder(file).Encode(data); err == nil { + err = file.Sync() + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + return err + } + return os.Rename(file.Name(), store.path) +} diff --git a/internal/app/helpers.go b/internal/app/helpers.go new file mode 100644 index 0000000..8cbcde7 --- /dev/null +++ b/internal/app/helpers.go @@ -0,0 +1,142 @@ +package app + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +var ( + badNameChars = regexp.MustCompile(`[\\/\?%\*:|"<>\x00-\x1f]+`) + spaceChars = regexp.MustCompile(`\s+`) + rePublicURL = regexp.MustCompile(`(?i)(?:https?|socks5h?)://[^\s"'<>]+`) +) + +func publicError(err error) error { + if err == nil { + return nil + } + return errors.New(redactErrorString(err.Error())) +} + +func redactErrorString(s string) string { + return rePublicURL.ReplaceAllStringFunc(s, func(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "[无效 URL 已隐藏]" + } + if u.Host == "" { + return raw + } + if u.User != nil { + u.User = url.User("redacted") + } + if u.RawQuery != "" { + u.RawQuery = "[redacted]" + } + if u.Fragment != "" { + u.Fragment = "redacted" + } + return u.String() + }) +} + +func saveFailures(outputDir string, results []Result) error { + var failed []Result + for _, r := range results { + if !r.OK { + failed = append(failed, r) + } + } + if len(failed) == 0 { + return nil + } + b, err := json.MarshalIndent(failed, "", " ") + if err != nil { + return err + } + return os.WriteFile(filepath.Join(outputDir, "failed.json"), b, 0o644) +} + +func existingGood(path string, skipBytes int64) (bool, int64) { + st, err := os.Stat(path) + if err != nil || st.IsDir() { + return false, 0 + } + return st.Size() >= skipBytes, st.Size() +} + +func safeFilename(name string) string { + name = strings.TrimSpace(name) + name = badNameChars.ReplaceAllString(name, "_") + name = strings.ReplaceAll(name, "..", "_") + name = spaceChars.ReplaceAllString(name, " ") + name = strings.Trim(name, " ._") + if name == "" { + return "未命名" + } + upper := strings.ToUpper(name) + reserved := map[string]bool{"CON": true, "PRN": true, "AUX": true, "NUL": true, "COM1": true, "COM2": true, "COM3": true, "COM4": true, "COM5": true, "COM6": true, "COM7": true, "COM8": true, "COM9": true, "LPT1": true, "LPT2": true, "LPT3": true, "LPT4": true, "LPT5": true, "LPT6": true, "LPT7": true, "LPT8": true, "LPT9": true} + if reserved[strings.SplitN(upper, ".", 2)[0]] { + name = "_" + name + } + runes := []rune(name) + if len(runes) > 100 { + name = string(runes[:100]) + } + return name +} + +func safeJoin(base string, elem ...string) (string, error) { + cleanBase, err := filepath.Abs(base) + if err != nil { + return "", err + } + parts := []string{cleanBase} + for _, e := range elem { + parts = append(parts, safeFilename(e)) + } + joined := filepath.Join(parts...) + abs, err := filepath.Abs(joined) + if err != nil { + return "", err + } + rel, err := filepath.Rel(cleanBase, abs) + if err != nil { + return "", err + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) { + return "", fmt.Errorf("unsafe path outside output dir: %s", abs) + } + return abs, nil +} + +func padEpisode(ep string) string { + ep = strings.TrimSpace(ep) + if n, err := strconv.Atoi(ep); err == nil { + return fmt.Sprintf("%03d", n) + } + return safeFilename(ep) +} + +func hashShort(s string) string { + h := sha256.Sum256([]byte(s)) + return hex.EncodeToString(h[:])[:12] +} + +func truncate(s string, n int) string { + s = strings.TrimSpace(s) + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) + "..." +} diff --git a/internal/app/hongguo_parity_test.go b/internal/app/hongguo_parity_test.go new file mode 100644 index 0000000..8939fa2 --- /dev/null +++ b/internal/app/hongguo_parity_test.go @@ -0,0 +1,106 @@ +package app + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +func parityRankingHTML(board rankingBoard, jsonLD bool) string { + header := `` + if jsonLD { + return header + `` + } + return header + `
` +} + +func TestParityHongguoRankingTextFallbackAndIdentity(t *testing.T) { + board, _ := findRankingBoard("hongguo-hot") + for _, jsonLD := range []bool{false, true} { + body := parityRankingHTML(board, jsonLD) + page, err := parseHongguoRanking(body, board, 1) + if err != nil || len(page.Items) != 1 || page.Items[0].Drama.ID != "hongguo:700001" || page.Items[0].Rank != 1 || page.HasMore { + t.Fatalf("text fallback failed (JSON-LD=%v): %+v %v", jsonLD, page, err) + } + if _, err := parseHongguoRanking(strings.ReplaceAll(body, board.path, "hot-ai-drama"), board, 1); err == nil { + t.Fatal("a different ranking board was accepted") + } + if _, err := parseHongguoRanking(body, board, 2); err == nil { + t.Fatal("a different page was accepted") + } + if _, err := parseHongguoRanking(strings.ReplaceAll(body, "700001", "invalid"), board, 1); err == nil { + t.Fatal("invalid drama identity was accepted") + } + } +} + +func TestParityHongguoRankingRetryIsBoundedAndBypassesCache(t *testing.T) { + board, _ := findRankingBoard("hongguo-hot") + var calls atomic.Int32 + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + count := calls.Add(1) + if request.URL.RawQuery != "" { + t.Error("first page is not canonical") + } + if count > 1 && (request.Header.Get("Cache-Control") != "no-cache" || request.Header.Get("Pragma") != "no-cache") { + t.Error("retry reused the incomplete cached page") + } + return rankingHTTPResponse(request, 200, ``), nil + }) + d.cfg.Retries = 10 + _, err := d.fetchHongguoRankingPage(context.Background(), board, 1) + if !errors.Is(err, errHongguoRankingIncomplete) || calls.Load() != 3 { + t.Fatalf("retry budget changed: %d %v", calls.Load(), err) + } +} + +func TestParityHongguoSearchDeadlineKeepsSharedPartialResult(t *testing.T) { + d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("unexpected network request: %s", request.URL.Path) + }) + pending := &hongguoSearchCall{done: make(chan struct{}), changed: make(chan struct{}), snapshot: hongguoSearchEntry{Dramas: []Drama{{ID: "hongguo:700001", Title: "合成结果"}}, Total: 1}} + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second)) + defer cancel() + result, err := d.waitHongguoSearch(ctx, pending, nil) + if err != nil || len(result.Dramas) != 1 || !result.Limited || result.Warning == "" { + t.Fatalf("deadline discarded partial result: %+v %v", result, err) + } + canceled, stop := context.WithCancel(context.Background()) + stop() + if _, err := d.waitHongguoSearch(canceled, pending, nil); !errors.Is(err, context.Canceled) { + t.Fatal("explicit cancellation became a successful search") + } +} + +func TestParityHongguoSearchSortKeepsFamiliesAndTokenMatches(t *testing.T) { + if hongguoTitleSearchRank("合成 第五季 故事", "合成 故事") >= hongguoTitleSearchRank("无关内容", "合成 故事") { + t.Fatal("separate title terms did not outrank unrelated results") + } + rows := []Drama{{Title: "合成故事第十季"}, {Title: "合成番外第二季"}, {Title: "合成故事第二季"}, {Title: "合成故事第一季"}} + sortHongguoSearchDramas(rows, "合成") + want := []string{"合成故事第一季", "合成番外第二季", "合成故事第二季", "合成故事第十季"} + for index, drama := range rows { + if drama.Title != want[index] { + t.Fatalf("cross-family order changed: %+v", rows) + } + } +} + +func TestParityHongguoMediaRetainsBackupsWithoutInventingPrimary(t *testing.T) { + primary := "https://media.example.test/main.mp4?token=a%2Fb" + backup := "https://backup.example.test/backup.mp4" + model := map[string]any{"video_list": []any{map[string]any{ + "main_url": "invalid", "backup_url": base64.StdEncoding.EncodeToString([]byte(primary)), "url_list": []any{primary, backup}, + "video_meta": map[string]any{"codec_type": "h264", "vheight": "720"}, + }}} + media, err := selectHongguoAppMedia(model) + if err != nil || media.URL != primary || len(media.Variants) != 2 || media.Variants[1].URL != backup || media.Quality != 720 { + t.Fatalf("backup media fields were lost: %+v %v", media, err) + } +} diff --git a/internal/app/image_heif.go b/internal/app/image_heif.go new file mode 100644 index 0000000..96c7ea4 --- /dev/null +++ b/internal/app/image_heif.go @@ -0,0 +1,360 @@ +package app + +import ( + "encoding/binary" + "errors" +) + +const maxCoverBytes = 20 << 20 + +type heifBox struct { + kind string + data []byte +} + +type heifReader struct { + data []byte + err error +} + +func (reader *heifReader) take(size int) []byte { + if reader.err != nil || size < 0 || size > len(reader.data) { + reader.err = errors.New("HEIC 数据不完整") + return nil + } + value := reader.data[:size] + reader.data = reader.data[size:] + return value +} + +func (reader *heifReader) uint(size int) uint64 { + if size < 0 || size > 8 { + reader.err = errors.New("HEIC 字段长度无效") + return 0 + } + var value uint64 + for _, part := range reader.take(size) { + value = value<<8 | uint64(part) + } + return value +} + +func heifBoxes(data []byte) ([]heifBox, error) { + var boxes []heifBox + for len(data) > 0 { + if len(data) < 8 || len(boxes) >= 4096 { + return nil, errors.New("HEIC 容器结构无效") + } + size, header := uint64(binary.BigEndian.Uint32(data)), 8 + if size == 1 { + if len(data) < 16 { + return nil, errors.New("HEIC 容器头不完整") + } + size, header = binary.BigEndian.Uint64(data[8:]), 16 + } else if size == 0 { + size = uint64(len(data)) + } + if size < uint64(header) || size > uint64(len(data)) { + return nil, errors.New("HEIC 容器长度无效") + } + boxes = append(boxes, heifBox{kind: string(data[4:8]), data: data[header:int(size)]}) + data = data[int(size):] + } + return boxes, nil +} + +func heifBoxData(boxes []heifBox, kind string) []byte { + for _, box := range boxes { + if box.kind == kind { + return box.data + } + } + return nil +} + +func isHEICImage(data []byte) bool { + if len(data) < 16 || string(data[4:8]) != "ftyp" { + return false + } + size := uint64(binary.BigEndian.Uint32(data)) + if size < 16 || size > uint64(len(data)) || size > 256 { + return false + } + for offset := 8; offset+4 <= int(size); offset += 4 { + if offset == 12 { + continue + } + switch string(data[offset : offset+4]) { + case "heic", "heix", "hevc", "hevx": + return true + } + } + return false +} + +type heifImage struct { + data []byte + filters []string +} + +func extractHEICImage(data []byte) (heifImage, error) { + var image heifImage + if len(data) > maxCoverBytes || !isHEICImage(data) { + return image, errors.New("不是支持的 HEIC 封面") + } + boxes, err := heifBoxes(data) + if err != nil { + return image, err + } + meta := heifBoxData(boxes, "meta") + if len(meta) < 4 || meta[0] != 0 { + return image, errors.New("HEIC 缺少图片元数据") + } + children, err := heifBoxes(meta[4:]) + if err != nil { + return image, err + } + primary := &heifReader{data: heifBoxData(children, "pitm")} + version := primary.uint(1) + primary.take(3) + idSize := 2 + if version == 1 { + idSize = 4 + } + itemID := primary.uint(idSize) + if primary.err != nil || version > 1 || itemID == 0 { + return image, errors.New("HEIC 主图片标识无效") + } + if err := validateHEICItem(heifBoxData(children, "iinf"), itemID); err != nil { + return image, err + } + config, filters, err := heifItemProperties(heifBoxData(children, "iprp"), itemID) + if err != nil { + return image, err + } + item, err := heifItemData(heifBoxData(children, "iloc"), itemID, data, heifBoxData(children, "idat")) + if err != nil { + return image, err + } + image.data, err = heifAnnexB(config, item) + image.filters = filters + return image, err +} + +func validateHEICItem(data []byte, itemID uint64) error { + reader := &heifReader{data: data} + version := reader.uint(1) + reader.take(3) + countSize := 2 + if version == 1 { + countSize = 4 + } + count := reader.uint(countSize) + if reader.err != nil || version > 1 { + return errors.New("HEIC 图片信息无效") + } + boxes, err := heifBoxes(reader.data) + if err != nil || count != uint64(len(boxes)) { + return errors.New("HEIC 图片信息不完整") + } + for _, box := range boxes { + if box.kind != "infe" { + continue + } + item := &heifReader{data: box.data} + version := item.uint(1) + item.take(3) + idSize := 2 + if version == 3 { + idSize = 4 + } + id := item.uint(idSize) + protected := item.uint(2) + kind := string(item.take(4)) + if id == itemID { + if item.err != nil || (version != 2 && version != 3) || protected != 0 || kind != "hvc1" { + return errors.New("HEIC 封面不是独立的 HEVC 图片") + } + return nil + } + } + return errors.New("HEIC 主图片不存在") +} + +func heifItemProperties(data []byte, itemID uint64) ([]byte, []string, error) { + boxes, err := heifBoxes(data) + if err != nil { + return nil, nil, err + } + properties, err := heifBoxes(heifBoxData(boxes, "ipco")) + if err != nil { + return nil, nil, err + } + reader := &heifReader{data: heifBoxData(boxes, "ipma")} + version, flags := reader.uint(1), reader.uint(3) + count := reader.uint(4) + if version > 1 || count > 4096 || reader.err != nil { + return nil, nil, errors.New("HEIC 图片属性无效") + } + var config []byte + var filters []string + var width, height uint64 + for entry := uint64(0); entry < count && reader.err == nil; entry++ { + idSize := 2 + if version == 1 { + idSize = 4 + } + id := reader.uint(idSize) + associations := reader.uint(1) + for association := uint64(0); association < associations && reader.err == nil; association++ { + index := reader.uint(1) & 127 + if flags&1 != 0 { + index = index<<8 | reader.uint(1) + } + if id != itemID || index == 0 { + continue + } + if index > uint64(len(properties)) { + return nil, nil, errors.New("HEIC 属性索引越界") + } + property := properties[index-1] + switch property.kind { + case "hvcC": + config = property.data + case "ispe": + if len(property.data) != 12 { + return nil, nil, errors.New("HEIC 图片尺寸无效") + } + width = uint64(binary.BigEndian.Uint32(property.data[4:])) + height = uint64(binary.BigEndian.Uint32(property.data[8:])) + case "irot", "imir": + if len(property.data) != 1 { + return nil, nil, errors.New("HEIC 图片方向无效") + } + if property.kind == "imir" { + if property.data[0]&1 == 0 { + filters = append(filters, "hflip") + } else { + filters = append(filters, "vflip") + } + } else { + switch property.data[0] & 3 { + case 1: + filters = append(filters, "transpose=cclock") + case 2: + filters = append(filters, "hflip", "vflip") + case 3: + filters = append(filters, "transpose=clock") + } + } + } + } + } + if reader.err != nil || len(config) < 23 || width == 0 || height == 0 || width > 4096 || height > 4096 { + return nil, nil, errors.New("HEIC 编码信息或图片尺寸无效") + } + return config, filters, nil +} + +func heifItemData(data []byte, itemID uint64, file, inline []byte) ([]byte, error) { + reader := &heifReader{data: data} + version := reader.uint(1) + reader.take(3) + lengths, offsets := reader.uint(1), reader.uint(1) + offsetSize, lengthSize := int(lengths>>4), int(lengths&15) + baseSize, indexSize := int(offsets>>4), 0 + if version > 0 { + indexSize = int(offsets & 15) + } + countSize := 2 + if version == 2 { + countSize = 4 + } + count := reader.uint(countSize) + if reader.err != nil || version > 2 || count > 4096 || offsetSize > 8 || lengthSize > 8 || baseSize > 8 || indexSize > 8 { + return nil, errors.New("HEIC 图片位置无效") + } + var result []byte + for entry := uint64(0); entry < count && reader.err == nil; entry++ { + id := reader.uint(countSize) + method := uint64(0) + if version > 0 { + method = reader.uint(2) & 15 + } + reference, base := reader.uint(2), reader.uint(baseSize) + extents := reader.uint(2) + if extents > 4096 || (id == itemID && (reference != 0 || method > 1)) { + return nil, errors.New("HEIC 不支持外部图片引用") + } + for extent := uint64(0); extent < extents && reader.err == nil; extent++ { + reader.uint(indexSize) + offset, length := reader.uint(offsetSize), reader.uint(lengthSize) + if id != itemID || reader.err != nil { + continue + } + source := file + if method == 1 { + source = inline + } + if base > uint64(len(source)) || offset > uint64(len(source))-base { + return nil, errors.New("HEIC 图片偏移越界") + } + start := base + offset + if length == 0 { + length = uint64(len(source)) - start + } + if length > uint64(len(source))-start || length > uint64(maxCoverBytes-len(result)) { + return nil, errors.New("HEIC 图片长度越界") + } + result = append(result, source[start:start+length]...) + } + } + if reader.err != nil || len(result) == 0 { + return nil, errors.New("HEIC 图片数据缺失") + } + return result, nil +} + +func heifAnnexB(config, item []byte) ([]byte, error) { + if len(config) < 23 || config[0] != 1 { + return nil, errors.New("HEIC 解码配置无效") + } + var result []byte + appendNAL := func(data []byte) error { + if len(data) < 2 || len(data) > maxCoverBytes-4-len(result) { + return errors.New("HEIC 编码数据长度无效") + } + result = append(result, 0, 0, 0, 1) + result = append(result, data...) + return nil + } + reader := &heifReader{data: config[23:]} + for array := 0; array < int(config[22]) && reader.err == nil; array++ { + reader.uint(1) + count := reader.uint(2) + if count > 4096 { + return nil, errors.New("HEIC 解码配置过大") + } + for index := uint64(0); index < count && reader.err == nil; index++ { + size := reader.uint(2) + if err := appendNAL(reader.take(int(size))); err != nil { + return nil, err + } + } + } + if reader.err != nil || len(reader.data) != 0 || len(result) == 0 { + return nil, errors.New("HEIC 解码配置不完整") + } + reader = &heifReader{data: item} + lengthSize := int(config[21]&3) + 1 + for count := 0; len(reader.data) > 0; count++ { + size := reader.uint(lengthSize) + if reader.err != nil || size > uint64(len(reader.data)) || count >= 4096 { + return nil, errors.New("HEIC 图片编码不完整") + } + if err := appendNAL(reader.take(int(size))); err != nil { + return nil, err + } + } + return result, nil +} diff --git a/internal/app/library.go b/internal/app/library.go new file mode 100644 index 0000000..94baca3 --- /dev/null +++ b/internal/app/library.go @@ -0,0 +1,59 @@ +package app + +import ( + "fmt" + "sort" + "strings" +) + +type libraryLoadError struct { + failures map[string]error +} + +func (failure *libraryLoadError) Error() string { + var messages []string + for source, cause := range failure.failures { + messages = append(messages, fmt.Sprintf("%s 获取失败: %v", source, publicError(cause))) + } + sort.Strings(messages) + return strings.Join(messages, "\n") +} + +func dramaProvider(drama Drama) string { + if source := canonicalProviderSource(drama.Source); source != "" { + return source + } + if source, _, ok := splitProviderDramaID(drama.ID); ok { + return source + } + return "" +} + +func mergeLoadedDramas(cached, fresh []Drama, loadErr error) []Drama { + return mergeSourceDramas(cached, fresh, loadErr, "") +} + +func matchesSourceFilter(provider, source string) bool { + return canonicalProviderSource(provider) == sourceHongguo && (source == "" || canonicalProviderSource(source) == sourceHongguo) +} + +func mergeSourceDramas(cached, fresh []Drama, loadErr error, source string) []Drama { + cached, fresh = onlyHongguoDramas(cached), onlyHongguoDramas(fresh) + merged := make([]Drama, 0, len(cached)+len(fresh)) + positions := make(map[string]int, cap(merged)) + for _, batch := range [][]Drama{cached, fresh} { + for _, drama := range batch { + if drama.ID == "" { + continue + } + if position, found := positions[drama.ID]; found { + merged[position] = mergeDramaMetadata(drama, merged[position]) + } else { + positions[drama.ID] = len(merged) + merged = append(merged, drama) + } + } + } + sort.SliceStable(merged, func(left, right int) bool { return merged[left].DisplayTitle() < merged[right].DisplayTitle() }) + return merged +}