jocay/ggjuku
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(app): add internal components batch 4

Browse Source
This commit is contained in:
吴璨
2026-09-30 10:20:08 +08:00
parent 713098f5b7
commit 4ec1f85bbc
10 changed files with 1374 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/app/ffmpeg_install_test.go
+309
View File
@@ -0,0 +1,309 @@
package app
import (
"bytes"
"compress/gzip"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
)
func TestMain(m *testing.M) {
if os.Getenv("JUKU_TEST_FFMPEG_HELPER") == "1" {
ffmpegTestHelper()
return
}
os.Exit(m.Run())
}
func ffmpegTestHelper() {
mode, _ := os.ReadFile(os.Args[0] + ".mode")
if log, err := os.OpenFile(os.Args[0]+".calls", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600); err == nil {
fmt.Fprintln(log, strings.Join(os.Args[1:], " "))
_ = log.Close()
}
if string(mode) == "hang" {
time.Sleep(time.Minute)
}
if string(mode) == "slow" {
time.Sleep(100 * time.Millisecond)
}
if strings.Contains(strings.Join(os.Args[1:], " "), "-encoders") {
fmt.Println(" A..... aac AAC")
if string(mode) != "missing" {
fmt.Println(" V..... libx264 H.264")
}
return
}
if string(mode) == "preset" {
fmt.Fprintln(os.Stderr, "Unrecognized option 'preset'.\r\nError splitting the argument list: Option not found")
os.Exit(8)
}
fmt.Print("synthetic MP4 output")
}
func ffmpegTestEnvironment(t *testing.T) string {
t.Helper()
previous, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
root := t.TempDir()
if err := os.Chdir(root); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = os.Chdir(previous) })
t.Setenv("PATH", "")
t.Setenv("JUKU_TEST_FFMPEG_HELPER", "1")
return root
}
func ffmpegTestProgram(t *testing.T, directory, mode string) string {
t.Helper()
if err := os.MkdirAll(directory, 0755); err != nil {
t.Fatal(err)
}
program, err := os.Executable()
if err != nil {
t.Fatal(err)
}
path := filepath.Join(directory, ffmpegExecutableName())
if err := os.Link(program, path); err != nil {
body, err := os.ReadFile(program)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, body, 0755); err != nil {
t.Fatal(err)
}
}
if err := os.WriteFile(path+".mode", []byte(mode), 0600); err != nil {
t.Fatal(err)
}
return path
}
func ffmpegTestInstaller(t *testing.T, configured string) *ffmpegInstaller {
t.Helper()
return &ffmpegInstaller{
configured: configured, name: ffmpegExecutableName(), directory: ffmpegManagedDirectory(),
client: &http.Client{Transport: rankingTransport(func(*http.Request) (*http.Response, error) {
t.Error("unexpected FFmpeg download")
return nil, errors.New("network forbidden")
})},
}
}
func TestFFmpegPATHPrecedesPortableAndKeepsAutomaticSetting(t *testing.T) {
root := ffmpegTestEnvironment(t)
ffmpegTestProgram(t, filepath.Join(root, "bin"), "preset")
ffmpegTestProgram(t, root, "preset")
full := ffmpegTestProgram(t, filepath.Join(root, "系统 PATH with spaces"), "good")
t.Setenv("PATH", filepath.Dir(full))
d := NewDownloader(Config{FFmpeg: "ffmpeg", OutputDir: filepath.Join(root, "out"), dataDir: filepath.Join(root, "data"), settingsLoaded: true})
if d.cfg.FFmpeg != "ffmpeg" {
t.Fatal("automatic discovery was frozen as an explicit portable path", d.cfg.FFmpeg)
}
if d.ffmpegInstallation().snapshot().Status == "ready" {
t.Fatal("unverified executable advertised as ready")
}
path, err := d.ensureFFmpeg(context.Background())
if err != nil || path != full {
t.Fatal("PATH must take precedence over current-directory and bundled programs", path, err)
}
if !strings.Contains(d.ffmpegInstallation().snapshot().Detail, full) {
t.Fatal("the selected executable must be visible in status")
}
}
func TestFFmpegSkipsIncompatiblePATHCandidates(t *testing.T) {
root := ffmpegTestEnvironment(t)
first := ffmpegTestProgram(t, filepath.Join(root, "minimal"), "missing")
second := ffmpegTestProgram(t, filepath.Join(root, "advertised-only"), "preset")
full := ffmpegTestProgram(t, filepath.Join(root, "full"), "good")
t.Setenv("PATH", strings.Join([]string{filepath.Dir(first), filepath.Dir(second), filepath.Dir(first), filepath.Dir(full)}, string(os.PathListSeparator)))
installer := ffmpegTestInstaller(t, "ffmpeg")
var events []diagnosticEvent
installer.record = func(event diagnosticEvent) { events = append(events, event) }
path, err := installer.ensure(context.Background())
if err != nil || path != full {
t.Fatal("a PATH hit without working encoders must not mask a usable fallback", path, err)
}
if len(events) != 3 || events[0].Event != "ffmpeg.rejected" || !strings.Contains(events[1].Message, "preset") || events[2].Event != "ffmpeg.ready" || !strings.Contains(events[2].Message, full) {
t.Fatal("selection diagnostics must explain rejected candidates and the final path", events)
}
if candidates := ffmpegCandidates("ffmpeg", installer.directory); len(candidates) != 3 {
t.Fatal("duplicate PATH entries were not deduplicated", candidates)
}
}
func TestFFmpegExplicitPathsRemainAuthoritative(t *testing.T) {
root := ffmpegTestEnvironment(t)
full := ffmpegTestProgram(t, filepath.Join(root, "full"), "good")
bad := ffmpegTestProgram(t, filepath.Join(root, "explicit"), "preset")
t.Setenv("PATH", filepath.Dir(full))
installer := ffmpegTestInstaller(t, bad)
path, err := installer.ensure(context.Background())
if err == nil || path != "" || !strings.Contains(err.Error(), "preset") || !strings.Contains(err.Error(), bad) {
t.Fatal("an invalid explicit path must report its failure without switching programs", path, err)
}
if installer.snapshot().Status != "failed" {
t.Fatal("invalid explicit program was marked ready")
}
if err := os.WriteFile(bad+".mode", []byte("good"), 0600); err != nil {
t.Fatal(err)
}
installer.retry()
path, err = installer.ensure(context.Background())
if err != nil || path != bad {
t.Fatal("retry must probe a corrected explicit program", path, err)
}
installer = ffmpegTestInstaller(t, " \""+full+"\" ")
path, err = installer.ensure(context.Background())
if err != nil || path != full {
t.Fatal("quoted paths with spaces were not resolved", path, err)
}
installer = ffmpegTestInstaller(t, filepath.Join(root, "missing", "ffmpeg.exe"))
if _, err := installer.ensure(context.Background()); err == nil || !strings.Contains(err.Error(), "找不到指定") {
t.Fatal("missing explicit path must not silently download", err)
}
}
func TestFFmpegSharedProbeIsCachedAndCancelable(t *testing.T) {
root := ffmpegTestEnvironment(t)
program := ffmpegTestProgram(t, filepath.Join(root, "slow"), "slow")
installer := ffmpegTestInstaller(t, program)
var wait sync.WaitGroup
for index := 0; index < 8; index++ {
wait.Add(1)
go func() {
defer wait.Done()
if path, err := installer.ensure(context.Background()); err != nil || path != program {
t.Error(path, err)
}
}()
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := installer.ensure(ctx); !errors.Is(err, context.Canceled) {
t.Fatal("canceled waiter did not exit", err)
}
wait.Wait()
if _, err := installer.ensure(context.Background()); err != nil {
t.Fatal(err)
}
log, err := os.ReadFile(program + ".calls")
if err != nil || len(strings.Split(strings.TrimSpace(string(log)), "\n")) != 2 {
t.Fatal("concurrent users must share one encoder query and one real probe", string(log), err)
}
hanging := ffmpegTestProgram(t, filepath.Join(root, "hang"), "hang")
ctx, cancel = context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
start := time.Now()
if err := validateFFmpeg(ctx, hanging); err == nil || time.Since(start) > 3*time.Second {
t.Fatal("capability probe must honor cancellation", err, time.Since(start))
}
}
func ffmpegTestPackage(t *testing.T, installer *ffmpegInstaller) *atomic.Int32 {
t.Helper()
program, err := os.Executable()
if err != nil {
t.Fatal(err)
}
binary, err := os.ReadFile(program)
if err != nil {
t.Fatal(err)
}
var archive bytes.Buffer
writer, _ := gzip.NewWriterLevel(&archive, gzip.BestSpeed)
_, _ = writer.Write(binary)
if err := writer.Close(); err != nil {
t.Fatal(err)
}
digest := func(body []byte) string { value := sha256.Sum256(body); return hex.EncodeToString(value[:]) }
document := []byte("synthetic test package documentation")
installer.pack = ffmpegPackage{platform: "fixture", archiveSize: int64(archive.Len()), binarySize: int64(len(binary)), archiveSHA256: digest(archive.Bytes()), binarySHA256: digest(binary), licenseSHA256: digest(document), readmeSHA256: digest(document)}
var requests atomic.Int32
installer.client = &http.Client{Transport: rankingTransport(func(request *http.Request) (*http.Response, error) {
requests.Add(1)
body := document
if strings.HasSuffix(request.URL.Path, ".gz") {
body = archive.Bytes()
}
return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(body)), ContentLength: int64(len(body)), Request: request}, nil
})}
return &requests
}
func TestFFmpegAutomaticFallbackPreservesExistingPortableProgram(t *testing.T) {
root := ffmpegTestEnvironment(t)
old := ffmpegTestProgram(t, filepath.Join(root, "bin", runtime.GOOS+"-"+runtime.GOARCH), "missing")
info, err := os.Stat(old)
if err != nil {
t.Fatal(err)
}
installer := ffmpegTestInstaller(t, "ffmpeg")
requests := ffmpegTestPackage(t, installer)
path, err := installer.ensure(context.Background())
if err != nil || path != filepath.Join(installer.directory, installer.name) || path == old {
t.Fatal("verified auto-download must be selected after rejecting the old portable program", path, err)
}
if requests.Load() != 3 {
t.Fatal("archive, license and build details must all be fetched", requests.Load())
}
after, err := os.Stat(old)
if err != nil || !os.SameFile(info, after) {
t.Fatal("existing portable program was replaced", err)
}
restarted := ffmpegTestInstaller(t, "ffmpeg")
if next, err := restarted.ensure(context.Background()); err != nil || next != path {
t.Fatal("restart must reuse the verified installation without downloading", next, err)
}
}
func TestFFmpegInstallationRejectsCorruptionAndInvalidDestination(t *testing.T) {
root := ffmpegTestEnvironment(t)
installer := ffmpegTestInstaller(t, "ffmpeg")
ffmpegTestPackage(t, installer)
installer.pack.archiveSHA256 = strings.Repeat("0", 64)
if _, err := installer.install(context.Background()); err == nil || !strings.Contains(err.Error(), "SHA-256") {
t.Fatal("corrupt download was accepted", err)
}
if _, err := os.Stat(installer.directory); !errors.Is(err, os.ErrNotExist) {
t.Fatal("a corrupt download must not create a usable installation", err)
}
ffmpegTestPackage(t, installer)
bad := ffmpegTestProgram(t, installer.directory, "preset")
if path, err := installer.install(context.Background()); err == nil || path != "" {
t.Fatal("an existing invalid destination must not turn an install failure into success", path, err)
}
if mode, err := os.ReadFile(bad + ".mode"); err != nil || string(mode) != "preset" {
t.Fatal("existing installation contents were changed", root, err)
}
}
func TestFFmpegLocalPlaybackProbe(t *testing.T) {
program, err := exec.LookPath("ffmpeg")
if err != nil {
t.Skip("local FFmpeg unavailable")
}
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
defer cancel()
if err := validateFFmpeg(ctx, program); err != nil {
t.Fatal(err)
}
}
internal/app/ffmpeg_media.go
+22
View File
@@ -0,0 +1,22 @@
package app
import (
"context"
"os"
"os/exec"
"strings"
)
func ffmpegMediaCommand(ctx context.Context, executable string, args ...string) *exec.Cmd {
command := exec.CommandContext(ctx, executable, args...)
command.Env = make([]string, 0, len(os.Environ()))
for _, entry := range os.Environ() {
key, _, _ := strings.Cut(entry, "=")
switch strings.ToLower(key) {
case "http_proxy", "https_proxy", "all_proxy":
continue
}
command.Env = append(command.Env, entry)
}
return command
}
internal/app/ffmpeg_media_test.go
+91
View File
@@ -0,0 +1,91 @@
package app
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"sync/atomic"
"testing"
"time"
)
func TestFFmpegLoopbackKeepsApplicationProxy(t *testing.T) {
fixture, session := nativePlaybackFixture(t, "3", "640x360")
video, err := os.ReadFile(session.tasks[0].OutPath)
if err != nil {
t.Fatal(err)
}
var inheritedRequests, mediaRequests atomic.Int32
inheritedProxy := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
inheritedRequests.Add(1)
http.Error(writer, "loopback must not reach the inherited proxy", http.StatusBadGateway)
}))
t.Cleanup(inheritedProxy.Close)
applicationProxy := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
if request.URL.Host != "media.example.invalid" || request.URL.Path != "/fixture.mp4" {
t.Error("application proxy received an unrelated request")
http.NotFound(writer, request)
return
}
mediaRequests.Add(1)
http.ServeContent(writer, request, "fixture.mp4", time.Time{}, bytes.NewReader(video))
}))
t.Cleanup(applicationProxy.Close)
for _, key := range []string{"http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"} {
t.Setenv(key, inheritedProxy.URL)
}
for _, key := range []string{"no_proxy", "NO_PROXY"} {
t.Setenv(key, "")
}
for _, mode := range []string{"MSE", "MSE remux", "native HLS", "download"} {
t.Run(mode, func(t *testing.T) {
app, _ := prefetchFixtureApp(t)
root := t.TempDir()
app.downloader = NewDownloader(Config{FFmpeg: fixture.downloader.cfg.FFmpeg, ProxyURL: applicationProxy.URL,
OutputDir: filepath.Join(root, "out"), dataDir: filepath.Join(root, "data"), settingsLoaded: true, Retries: 1, SkipBytes: 1})
app.cfg = app.downloader.cfg
task := Task{DramaID: "hongguo:7000000000000000001", DramaTitle: "合成代理样本", Index: 1,
OutPath: filepath.Join(root, "out", "episode.mp4"), Chapter: Chapter{ID: "hongguo:7000000000000000001:1", Source: sourceHongguo,
VideoURL: "http://media.example.invalid/fixture.mp4"}}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
before := mediaRequests.Load()
switch mode {
case "MSE", "MSE remux":
ctx = context.WithValue(ctx, playbackRemuxKey{}, mode == "MSE remux")
writer := httptest.NewRecorder()
if err := app.streamPlayback(ctx, cancel, writer, task, "", 0, 1, func(float64) {}); err != nil {
t.Fatal("playback through the application proxy failed", err)
}
if !bytes.Contains(writer.Body.Bytes(), []byte("moof")) {
t.Fatal("playback did not return fragmented video")
}
case "native HLS":
cache := newPlaybackNative(app, ctx, task, "", 0, 0, false)
defer func() {
cache.Close()
cache.workers.Wait()
}()
cache.start()
body, err := cache.segment(ctx, 0)
if err != nil || len(body) < 188 || body[0] != 0x47 {
t.Fatal("native playback through the application proxy failed", err)
}
case "download":
if err := app.downloader.DownloadEpisode(ctx, task); err != nil {
t.Fatal("download through the application proxy failed", err)
}
info, err := os.Stat(task.OutPath)
if err != nil || info.Size() < 100*1024 {
t.Fatal("download did not produce the fixture", err)
}
}
if inheritedRequests.Load() != 0 || mediaRequests.Load() <= before {
t.Fatalf("incorrect proxy route: inherited=%d media=%d", inheritedRequests.Load(), mediaRequests.Load())
}
})
}
}
internal/app/ffmpeg_paths.go
+80
View File
@@ -0,0 +1,80 @@
package app
import (
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
)
func normalizedFFmpegSetting(value string) string {
value = strings.TrimSpace(value)
if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' {
value = strings.TrimSpace(value[1 : len(value)-1])
}
return value
}
func automaticFFmpeg(value string) bool {
value = normalizedFFmpegSetting(value)
if runtime.GOOS == "windows" {
value = strings.ToLower(value)
}
return value == "" || value == "ffmpeg" || value == "ffmpeg.exe"
}
func ffmpegExecutableName() string {
if runtime.GOOS == "windows" {
return "ffmpeg.exe"
}
return "ffmpeg"
}
func ffmpegManagedDirectory() string {
directory := filepath.Join("bin", runtime.GOOS+"-"+runtime.GOARCH, "verified-b6.1.1")
if absolute, err := filepath.Abs(directory); err == nil {
return absolute
}
return directory
}
func ffmpegCandidates(configured, managedDirectory string) []string {
var candidates []string
seen := make(map[string]bool)
add := func(candidate string) {
path, err := exec.LookPath(candidate)
if err != nil {
return
}
path, err = filepath.Abs(path)
if err != nil {
return
}
key := path
if runtime.GOOS == "windows" {
key = strings.ToLower(key)
}
if !seen[key] {
seen[key] = true
candidates = append(candidates, path)
}
}
configured = normalizedFFmpegSetting(configured)
if !automaticFFmpeg(configured) {
add(configured)
return candidates
}
name := ffmpegExecutableName()
for _, directory := range filepath.SplitList(os.Getenv("PATH")) {
if filepath.IsAbs(directory) {
add(filepath.Join(directory, name))
}
}
for _, path := range []string{filepath.Join("bin", name), name, filepath.Join(managedDirectory, name), filepath.Join("bin", runtime.GOOS+"-"+runtime.GOARCH, name)} {
if absolute, err := filepath.Abs(path); err == nil {
add(absolute)
}
}
return candidates
}
internal/app/ffmpeg_probe_windows.go
+12
View File
@@ -0,0 +1,12 @@
package app
import (
"os/exec"
"syscall"
)
func init() {
configureFFmpegProbeProcess = func(command *exec.Cmd) {
command.SysProcAttr = &syscall.SysProcAttr{HideWindow: true, CreationFlags: 0x08000000}
}
}
internal/app/following_store.go
+193
View File
@@ -0,0 +1,193 @@
package app
import (
"encoding/json"
"errors"
"io"
"os"
"path/filepath"
"sort"
"sync"
"time"
)
const followingLimit = 1000
var errFollowingLimit = errors.New("追剧清单已达到 1000 部，请先移除不再需要的内容")
var errFollowingUnknown = errors.New("剧库中没有这部剧，请先搜索或更新剧库")
type followingEntry struct {
DramaID string `json:"dramaId"`
Source string `json:"source"`
Title string `json:"title"`
Category string `json:"category,omitempty"`
Saved bool `json:"saved"`
Completed bool `json:"completed"`
KnownEpisodes int `json:"knownEpisodes"`
AddedAt time.Time `json:"addedAt"`
UpdatedAt time.Time `json:"updatedAt"`
}
type followingFile struct {
Version int `json:"version"`
Entries []followingEntry `json:"entries"`
}
type followingStore struct {
mu sync.Mutex
path string
entries map[string]followingEntry
loadErr error
}
func newFollowingStore(directory string) *followingStore {
store := &followingStore{path: filepath.Join(directory, "following.json"), entries: map[string]followingEntry{}}
file, err := os.Open(store.path)
if errors.Is(err, os.ErrNotExist) {
return store
}
if err != nil {
store.loadErr = err
return store
}
defer file.Close()
var saved followingFile
decoder := json.NewDecoder(io.LimitReader(file, 8<<20))
if err = decoder.Decode(&saved); err == nil {
if decoder.Decode(new(any)) != io.EOF || saved.Version != 1 || len(saved.Entries) > followingLimit {
err = errors.New("追剧清单格式无效，原文件已保留")
}
}
if err != nil {
store.loadErr = err
return store
}
for _, entry := range saved.Entries {
entry, valid := normalizeFollowingEntry(entry)
if !valid {
continue
}
if old, exists := store.entries[entry.DramaID]; !exists || entry.UpdatedAt.After(old.UpdatedAt) {
store.entries[entry.DramaID] = entry
}
}
return store
}
func followingEntries(entries map[string]followingEntry) []followingEntry {
result := make([]followingEntry, 0, len(entries))
for _, entry := range entries {
result = append(result, entry)
}
sort.Slice(result, func(i, j int) bool {
if result[i].UpdatedAt.Equal(result[j].UpdatedAt) {
return result[i].DramaID < result[j].DramaID
}
return result[i].UpdatedAt.After(result[j].UpdatedAt)
})
return result
}
func normalizeFollowingEntry(entry followingEntry) (followingEntry, bool) {
id, source, valid := playbackHistoryIdentity(entry.DramaID)
if !valid || canonicalProviderSource(entry.Source) != source || len(entry.Title) > 4096 || len(entry.Category) > 512 || entry.KnownEpisodes < 0 || entry.KnownEpisodes > 10000 || entry.AddedAt.IsZero() || entry.UpdatedAt.IsZero() || !entry.Saved && !entry.Completed {
return followingEntry{}, false
}
entry.DramaID, entry.Source = id, source
return entry, true
}
func (store *followingStore) list() ([]followingEntry, error) {
store.mu.Lock()
defer store.mu.Unlock()
return followingEntries(store.entries), store.loadErr
}
func (store *followingStore) update(id string, change func(followingEntry, bool) (followingEntry, error)) (followingEntry, error) {
store.mu.Lock()
defer store.mu.Unlock()
if store.loadErr != nil {
return followingEntry{}, store.loadErr
}
previous, exists := store.entries[id]
entry, err := change(previous, exists)
if err != nil {
return followingEntry{}, err
}
if !exists && (entry.Saved || entry.Completed) && len(store.entries) >= followingLimit {
return followingEntry{}, errFollowingLimit
}
next := make(map[string]followingEntry, len(store.entries)+1)
for key, value := range store.entries {
next[key] = value
}
if entry.Saved || entry.Completed {
next[id] = entry
} else {
delete(next, id)
}
if err = store.write(followingFile{Version: 1, Entries: followingEntries(next)}); err != nil {
return followingEntry{}, err
}
store.entries = next
return entry, nil
}
func (store *followingStore) merge(entries []followingEntry) (int, int, error) {
store.mu.Lock()
defer store.mu.Unlock()
if store.loadErr != nil {
return 0, 0, store.loadErr
}
next := make(map[string]followingEntry, len(store.entries)+len(entries))
for id, entry := range store.entries {
next[id] = entry
}
imported := 0
skipped := 0
for _, entry := range entries {
entry, valid := normalizeFollowingEntry(entry)
if !valid {
continue
}
current, exists := next[entry.DramaID]
if exists && !entry.UpdatedAt.After(current.UpdatedAt) {
continue
}
if !exists && len(next) >= followingLimit {
skipped++
continue
}
next[entry.DramaID] = entry
imported++
}
if imported == 0 {
return 0, skipped, nil
}
if err := store.write(followingFile{Version: 1, Entries: followingEntries(next)}); err != nil {
return 0, skipped, err
}
store.entries = next
return imported, skipped, nil
}
func (store *followingStore) write(data followingFile) error {
if err := os.MkdirAll(filepath.Dir(store.path), 0o700); err != nil {
return err
}
file, err := os.CreateTemp(filepath.Dir(store.path), ".following-*.json")
if err != nil {
return err
}
defer os.Remove(file.Name())
if err = json.NewEncoder(file).Encode(data); err == nil {
err = file.Sync()
}
if closeErr := file.Close(); err == nil {
err = closeErr
}
if err != nil {
return err
}
return os.Rename(file.Name(), store.path)
}
internal/app/helpers.go
+142
View File
@@ -0,0 +1,142 @@
package app
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
)
var (
badNameChars = regexp.MustCompile(`[\\/\?%\*:|"<>\x00-\x1f]+`)
spaceChars = regexp.MustCompile(`\s+`)
rePublicURL = regexp.MustCompile(`(?i)(?:https?|socks5h?)://[^\s"'<>]+`)
)
func publicError(err error) error {
if err == nil {
return nil
}
return errors.New(redactErrorString(err.Error()))
}
func redactErrorString(s string) string {
return rePublicURL.ReplaceAllStringFunc(s, func(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return "[无效 URL 已隐藏]"
}
if u.Host == "" {
return raw
}
if u.User != nil {
u.User = url.User("redacted")
}
if u.RawQuery != "" {
u.RawQuery = "[redacted]"
}
if u.Fragment != "" {
u.Fragment = "redacted"
}
return u.String()
})
}
func saveFailures(outputDir string, results []Result) error {
var failed []Result
for _, r := range results {
if !r.OK {
failed = append(failed, r)
}
}
if len(failed) == 0 {
return nil
}
b, err := json.MarshalIndent(failed, "", " ")
if err != nil {
return err
}
return os.WriteFile(filepath.Join(outputDir, "failed.json"), b, 0o644)
}
func existingGood(path string, skipBytes int64) (bool, int64) {
st, err := os.Stat(path)
if err != nil || st.IsDir() {
return false, 0
}
return st.Size() >= skipBytes, st.Size()
}
func safeFilename(name string) string {
name = strings.TrimSpace(name)
name = badNameChars.ReplaceAllString(name, "_")
name = strings.ReplaceAll(name, "..", "_")
name = spaceChars.ReplaceAllString(name, " ")
name = strings.Trim(name, " ._")
if name == "" {
return "未命名"
}
upper := strings.ToUpper(name)
reserved := map[string]bool{"CON": true, "PRN": true, "AUX": true, "NUL": true, "COM1": true, "COM2": true, "COM3": true, "COM4": true, "COM5": true, "COM6": true, "COM7": true, "COM8": true, "COM9": true, "LPT1": true, "LPT2": true, "LPT3": true, "LPT4": true, "LPT5": true, "LPT6": true, "LPT7": true, "LPT8": true, "LPT9": true}
if reserved[strings.SplitN(upper, ".", 2)[0]] {
name = "_" + name
}
runes := []rune(name)
if len(runes) > 100 {
name = string(runes[:100])
}
return name
}
func safeJoin(base string, elem ...string) (string, error) {
cleanBase, err := filepath.Abs(base)
if err != nil {
return "", err
}
parts := []string{cleanBase}
for _, e := range elem {
parts = append(parts, safeFilename(e))
}
joined := filepath.Join(parts...)
abs, err := filepath.Abs(joined)
if err != nil {
return "", err
}
rel, err := filepath.Rel(cleanBase, abs)
if err != nil {
return "", err
}
if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || filepath.IsAbs(rel) {
return "", fmt.Errorf("unsafe path outside output dir: %s", abs)
}
return abs, nil
}
func padEpisode(ep string) string {
ep = strings.TrimSpace(ep)
if n, err := strconv.Atoi(ep); err == nil {
return fmt.Sprintf("%03d", n)
}
return safeFilename(ep)
}
func hashShort(s string) string {
h := sha256.Sum256([]byte(s))
return hex.EncodeToString(h[:])[:12]
}
func truncate(s string, n int) string {
s = strings.TrimSpace(s)
r := []rune(s)
if len(r) <= n {
return s
}
return string(r[:n]) + "..."
}
internal/app/hongguo_parity_test.go
+106
View File
@@ -0,0 +1,106 @@
package app
import (
"context"
"encoding/base64"
"errors"
"fmt"
"net/http"
"strings"
"sync/atomic"
"testing"
"time"
)
func parityRankingHTML(board rankingBoard, jsonLD bool) string {
header := `<link rel="canonical" href="https://www.hongguoduanju.com/rank/` + board.path + `"><nav aria-label="榜单分页"><span aria-current="page">1</span></nav>`
if jsonLD {
return header + `<script type="application/ld+json">{"@type":"ItemList","url":"/rank/` + board.path + `","itemListElement":[{"position":1,"name":"合成榜单剧","url":"/detail?series_id=700001"}]}</script>`
}
return header + `<ol aria-label="热播榜"><li><article aria-labelledby="rank-title-700001"><a aria-label="查看短剧" href="/detail?series_id=700001">1</a><a href="/detail?series_id=700001"><h2 id="rank-title-700001">合成榜单剧</h2></a><span>12.3万热度</span></article></li></ol>`
}
func TestParityHongguoRankingTextFallbackAndIdentity(t *testing.T) {
board, _ := findRankingBoard("hongguo-hot")
for _, jsonLD := range []bool{false, true} {
body := parityRankingHTML(board, jsonLD)
page, err := parseHongguoRanking(body, board, 1)
if err != nil || len(page.Items) != 1 || page.Items[0].Drama.ID != "hongguo:700001" || page.Items[0].Rank != 1 || page.HasMore {
t.Fatalf("text fallback failed (JSON-LD=%v): %+v %v", jsonLD, page, err)
}
if _, err := parseHongguoRanking(strings.ReplaceAll(body, board.path, "hot-ai-drama"), board, 1); err == nil {
t.Fatal("a different ranking board was accepted")
}
if _, err := parseHongguoRanking(body, board, 2); err == nil {
t.Fatal("a different page was accepted")
}
if _, err := parseHongguoRanking(strings.ReplaceAll(body, "700001", "invalid"), board, 1); err == nil {
t.Fatal("invalid drama identity was accepted")
}
}
}
func TestParityHongguoRankingRetryIsBoundedAndBypassesCache(t *testing.T) {
board, _ := findRankingBoard("hongguo-hot")
var calls atomic.Int32
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
count := calls.Add(1)
if request.URL.RawQuery != "" {
t.Error("first page is not canonical")
}
if count > 1 && (request.Header.Get("Cache-Control") != "no-cache" || request.Header.Get("Pragma") != "no-cache") {
t.Error("retry reused the incomplete cached page")
}
return rankingHTTPResponse(request, 200, `<link rel="canonical" href="/rank/hot-drama">`), nil
})
d.cfg.Retries = 10
_, err := d.fetchHongguoRankingPage(context.Background(), board, 1)
if !errors.Is(err, errHongguoRankingIncomplete) || calls.Load() != 3 {
t.Fatalf("retry budget changed: %d %v", calls.Load(), err)
}
}
func TestParityHongguoSearchDeadlineKeepsSharedPartialResult(t *testing.T) {
d := rankingTestDownloader(t, func(request *http.Request) (*http.Response, error) {
return nil, fmt.Errorf("unexpected network request: %s", request.URL.Path)
})
pending := &hongguoSearchCall{done: make(chan struct{}), changed: make(chan struct{}), snapshot: hongguoSearchEntry{Dramas: []Drama{{ID: "hongguo:700001", Title: "合成结果"}}, Total: 1}}
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(-time.Second))
defer cancel()
result, err := d.waitHongguoSearch(ctx, pending, nil)
if err != nil || len(result.Dramas) != 1 || !result.Limited || result.Warning == "" {
t.Fatalf("deadline discarded partial result: %+v %v", result, err)
}
canceled, stop := context.WithCancel(context.Background())
stop()
if _, err := d.waitHongguoSearch(canceled, pending, nil); !errors.Is(err, context.Canceled) {
t.Fatal("explicit cancellation became a successful search")
}
}
func TestParityHongguoSearchSortKeepsFamiliesAndTokenMatches(t *testing.T) {
if hongguoTitleSearchRank("合成 第五季 故事", "合成 故事") >= hongguoTitleSearchRank("无关内容", "合成 故事") {
t.Fatal("separate title terms did not outrank unrelated results")
}
rows := []Drama{{Title: "合成故事第十季"}, {Title: "合成番外第二季"}, {Title: "合成故事第二季"}, {Title: "合成故事第一季"}}
sortHongguoSearchDramas(rows, "合成")
want := []string{"合成故事第一季", "合成番外第二季", "合成故事第二季", "合成故事第十季"}
for index, drama := range rows {
if drama.Title != want[index] {
t.Fatalf("cross-family order changed: %+v", rows)
}
}
}
func TestParityHongguoMediaRetainsBackupsWithoutInventingPrimary(t *testing.T) {
primary := "https://media.example.test/main.mp4?token=a%2Fb"
backup := "https://backup.example.test/backup.mp4"
model := map[string]any{"video_list": []any{map[string]any{
"main_url": "invalid", "backup_url": base64.StdEncoding.EncodeToString([]byte(primary)), "url_list": []any{primary, backup},
"video_meta": map[string]any{"codec_type": "h264", "vheight": "720"},
}}}
media, err := selectHongguoAppMedia(model)
if err != nil || media.URL != primary || len(media.Variants) != 2 || media.Variants[1].URL != backup || media.Quality != 720 {
t.Fatalf("backup media fields were lost: %+v %v", media, err)
}
}
internal/app/image_heif.go
+360
View File
@@ -0,0 +1,360 @@
package app
import (
"encoding/binary"
"errors"
)
const maxCoverBytes = 20 << 20
type heifBox struct {
kind string
data []byte
}
type heifReader struct {
data []byte
err error
}
func (reader *heifReader) take(size int) []byte {
if reader.err != nil || size < 0 || size > len(reader.data) {
reader.err = errors.New("HEIC 数据不完整")
return nil
}
value := reader.data[:size]
reader.data = reader.data[size:]
return value
}
func (reader *heifReader) uint(size int) uint64 {
if size < 0 || size > 8 {
reader.err = errors.New("HEIC 字段长度无效")
return 0
}
var value uint64
for _, part := range reader.take(size) {
value = value<<8 | uint64(part)
}
return value
}
func heifBoxes(data []byte) ([]heifBox, error) {
var boxes []heifBox
for len(data) > 0 {
if len(data) < 8 || len(boxes) >= 4096 {
return nil, errors.New("HEIC 容器结构无效")
}
size, header := uint64(binary.BigEndian.Uint32(data)), 8
if size == 1 {
if len(data) < 16 {
return nil, errors.New("HEIC 容器头不完整")
}
size, header = binary.BigEndian.Uint64(data[8:]), 16
} else if size == 0 {
size = uint64(len(data))
}
if size < uint64(header) || size > uint64(len(data)) {
return nil, errors.New("HEIC 容器长度无效")
}
boxes = append(boxes, heifBox{kind: string(data[4:8]), data: data[header:int(size)]})
data = data[int(size):]
}
return boxes, nil
}
func heifBoxData(boxes []heifBox, kind string) []byte {
for _, box := range boxes {
if box.kind == kind {
return box.data
}
}
return nil
}
func isHEICImage(data []byte) bool {
if len(data) < 16 || string(data[4:8]) != "ftyp" {
return false
}
size := uint64(binary.BigEndian.Uint32(data))
if size < 16 || size > uint64(len(data)) || size > 256 {
return false
}
for offset := 8; offset+4 <= int(size); offset += 4 {
if offset == 12 {
continue
}
switch string(data[offset : offset+4]) {
case "heic", "heix", "hevc", "hevx":
return true
}
}
return false
}
type heifImage struct {
data []byte
filters []string
}
func extractHEICImage(data []byte) (heifImage, error) {
var image heifImage
if len(data) > maxCoverBytes || !isHEICImage(data) {
return image, errors.New("不是支持的 HEIC 封面")
}
boxes, err := heifBoxes(data)
if err != nil {
return image, err
}
meta := heifBoxData(boxes, "meta")
if len(meta) < 4 || meta[0] != 0 {
return image, errors.New("HEIC 缺少图片元数据")
}
children, err := heifBoxes(meta[4:])
if err != nil {
return image, err
}
primary := &heifReader{data: heifBoxData(children, "pitm")}
version := primary.uint(1)
primary.take(3)
idSize := 2
if version == 1 {
idSize = 4
}
itemID := primary.uint(idSize)
if primary.err != nil || version > 1 || itemID == 0 {
return image, errors.New("HEIC 主图片标识无效")
}
if err := validateHEICItem(heifBoxData(children, "iinf"), itemID); err != nil {
return image, err
}
config, filters, err := heifItemProperties(heifBoxData(children, "iprp"), itemID)
if err != nil {
return image, err
}
item, err := heifItemData(heifBoxData(children, "iloc"), itemID, data, heifBoxData(children, "idat"))
if err != nil {
return image, err
}
image.data, err = heifAnnexB(config, item)
image.filters = filters
return image, err
}
func validateHEICItem(data []byte, itemID uint64) error {
reader := &heifReader{data: data}
version := reader.uint(1)
reader.take(3)
countSize := 2
if version == 1 {
countSize = 4
}
count := reader.uint(countSize)
if reader.err != nil || version > 1 {
return errors.New("HEIC 图片信息无效")
}
boxes, err := heifBoxes(reader.data)
if err != nil || count != uint64(len(boxes)) {
return errors.New("HEIC 图片信息不完整")
}
for _, box := range boxes {
if box.kind != "infe" {
continue
}
item := &heifReader{data: box.data}
version := item.uint(1)
item.take(3)
idSize := 2
if version == 3 {
idSize = 4
}
id := item.uint(idSize)
protected := item.uint(2)
kind := string(item.take(4))
if id == itemID {
if item.err != nil || (version != 2 && version != 3) || protected != 0 || kind != "hvc1" {
return errors.New("HEIC 封面不是独立的 HEVC 图片")
}
return nil
}
}
return errors.New("HEIC 主图片不存在")
}
func heifItemProperties(data []byte, itemID uint64) ([]byte, []string, error) {
boxes, err := heifBoxes(data)
if err != nil {
return nil, nil, err
}
properties, err := heifBoxes(heifBoxData(boxes, "ipco"))
if err != nil {
return nil, nil, err
}
reader := &heifReader{data: heifBoxData(boxes, "ipma")}
version, flags := reader.uint(1), reader.uint(3)
count := reader.uint(4)
if version > 1 || count > 4096 || reader.err != nil {
return nil, nil, errors.New("HEIC 图片属性无效")
}
var config []byte
var filters []string
var width, height uint64
for entry := uint64(0); entry < count && reader.err == nil; entry++ {
idSize := 2
if version == 1 {
idSize = 4
}
id := reader.uint(idSize)
associations := reader.uint(1)
for association := uint64(0); association < associations && reader.err == nil; association++ {
index := reader.uint(1) & 127
if flags&1 != 0 {
index = index<<8 | reader.uint(1)
}
if id != itemID || index == 0 {
continue
}
if index > uint64(len(properties)) {
return nil, nil, errors.New("HEIC 属性索引越界")
}
property := properties[index-1]
switch property.kind {
case "hvcC":
config = property.data
case "ispe":
if len(property.data) != 12 {
return nil, nil, errors.New("HEIC 图片尺寸无效")
}
width = uint64(binary.BigEndian.Uint32(property.data[4:]))
height = uint64(binary.BigEndian.Uint32(property.data[8:]))
case "irot", "imir":
if len(property.data) != 1 {
return nil, nil, errors.New("HEIC 图片方向无效")
}
if property.kind == "imir" {
if property.data[0]&1 == 0 {
filters = append(filters, "hflip")
} else {
filters = append(filters, "vflip")
}
} else {
switch property.data[0] & 3 {
case 1:
filters = append(filters, "transpose=cclock")
case 2:
filters = append(filters, "hflip", "vflip")
case 3:
filters = append(filters, "transpose=clock")
}
}
}
}
}
if reader.err != nil || len(config) < 23 || width == 0 || height == 0 || width > 4096 || height > 4096 {
return nil, nil, errors.New("HEIC 编码信息或图片尺寸无效")
}
return config, filters, nil
}
func heifItemData(data []byte, itemID uint64, file, inline []byte) ([]byte, error) {
reader := &heifReader{data: data}
version := reader.uint(1)
reader.take(3)
lengths, offsets := reader.uint(1), reader.uint(1)
offsetSize, lengthSize := int(lengths>>4), int(lengths&15)
baseSize, indexSize := int(offsets>>4), 0
if version > 0 {
indexSize = int(offsets & 15)
}
countSize := 2
if version == 2 {
countSize = 4
}
count := reader.uint(countSize)
if reader.err != nil || version > 2 || count > 4096 || offsetSize > 8 || lengthSize > 8 || baseSize > 8 || indexSize > 8 {
return nil, errors.New("HEIC 图片位置无效")
}
var result []byte
for entry := uint64(0); entry < count && reader.err == nil; entry++ {
id := reader.uint(countSize)
method := uint64(0)
if version > 0 {
method = reader.uint(2) & 15
}
reference, base := reader.uint(2), reader.uint(baseSize)
extents := reader.uint(2)
if extents > 4096 || (id == itemID && (reference != 0 || method > 1)) {
return nil, errors.New("HEIC 不支持外部图片引用")
}
for extent := uint64(0); extent < extents && reader.err == nil; extent++ {
reader.uint(indexSize)
offset, length := reader.uint(offsetSize), reader.uint(lengthSize)
if id != itemID || reader.err != nil {
continue
}
source := file
if method == 1 {
source = inline
}
if base > uint64(len(source)) || offset > uint64(len(source))-base {
return nil, errors.New("HEIC 图片偏移越界")
}
start := base + offset
if length == 0 {
length = uint64(len(source)) - start
}
if length > uint64(len(source))-start || length > uint64(maxCoverBytes-len(result)) {
return nil, errors.New("HEIC 图片长度越界")
}
result = append(result, source[start:start+length]...)
}
}
if reader.err != nil || len(result) == 0 {
return nil, errors.New("HEIC 图片数据缺失")
}
return result, nil
}
func heifAnnexB(config, item []byte) ([]byte, error) {
if len(config) < 23 || config[0] != 1 {
return nil, errors.New("HEIC 解码配置无效")
}
var result []byte
appendNAL := func(data []byte) error {
if len(data) < 2 || len(data) > maxCoverBytes-4-len(result) {
return errors.New("HEIC 编码数据长度无效")
}
result = append(result, 0, 0, 0, 1)
result = append(result, data...)
return nil
}
reader := &heifReader{data: config[23:]}
for array := 0; array < int(config[22]) && reader.err == nil; array++ {
reader.uint(1)
count := reader.uint(2)
if count > 4096 {
return nil, errors.New("HEIC 解码配置过大")
}
for index := uint64(0); index < count && reader.err == nil; index++ {
size := reader.uint(2)
if err := appendNAL(reader.take(int(size))); err != nil {
return nil, err
}
}
}
if reader.err != nil || len(reader.data) != 0 || len(result) == 0 {
return nil, errors.New("HEIC 解码配置不完整")
}
reader = &heifReader{data: item}
lengthSize := int(config[21]&3) + 1
for count := 0; len(reader.data) > 0; count++ {
size := reader.uint(lengthSize)
if reader.err != nil || size > uint64(len(reader.data)) || count >= 4096 {
return nil, errors.New("HEIC 图片编码不完整")
}
if err := appendNAL(reader.take(int(size))); err != nil {
return nil, err
}
}
return result, nil
}
internal/app/library.go
+59
View File
@@ -0,0 +1,59 @@
package app
import (
"fmt"
"sort"
"strings"
)
type libraryLoadError struct {
failures map[string]error
}
func (failure *libraryLoadError) Error() string {
var messages []string
for source, cause := range failure.failures {
messages = append(messages, fmt.Sprintf("%s 获取失败: %v", source, publicError(cause)))
}
sort.Strings(messages)
return strings.Join(messages, "\n")
}
func dramaProvider(drama Drama) string {
if source := canonicalProviderSource(drama.Source); source != "" {
return source
}
if source, _, ok := splitProviderDramaID(drama.ID); ok {
return source
}
return ""
}
func mergeLoadedDramas(cached, fresh []Drama, loadErr error) []Drama {
return mergeSourceDramas(cached, fresh, loadErr, "")
}
func matchesSourceFilter(provider, source string) bool {
return canonicalProviderSource(provider) == sourceHongguo && (source == "" || canonicalProviderSource(source) == sourceHongguo)
}
func mergeSourceDramas(cached, fresh []Drama, loadErr error, source string) []Drama {
cached, fresh = onlyHongguoDramas(cached), onlyHongguoDramas(fresh)
merged := make([]Drama, 0, len(cached)+len(fresh))
positions := make(map[string]int, cap(merged))
for _, batch := range [][]Drama{cached, fresh} {
for _, drama := range batch {
if drama.ID == "" {
continue
}
if position, found := positions[drama.ID]; found {
merged[position] = mergeDramaMetadata(drama, merged[position])
} else {
positions[drama.ID] = len(merged)
merged = append(merged, drama)
}
}
}
sort.SliceStable(merged, func(left, right int) bool { return merged[left].DisplayTitle() < merged[right].DisplayTitle() })
return merged
}