From 681225c2ae18aad2fbaa8ff64e2b6d83f710d521 Mon Sep 17 00:00:00 2001 From: Evan Fiordeliso Date: Wed, 6 Nov 2024 22:55:47 -0500 Subject: [PATCH] Fix config loading and fix/simplify yt-dlp command handling --- app/controllers/download.go | 35 +++++-------- config/config.go | 51 ++++++++++-------- pkg/ytdl/cmdbuilder.go | 56 -------------------- pkg/ytdl/cmdopts.go | 99 ---------------------------------- pkg/ytdl/error.go | 14 ----- pkg/ytdl/ytdl.go | 102 +++++++++++++++++++++++++----------- 6 files changed, 112 insertions(+), 245 deletions(-) delete mode 100644 pkg/ytdl/cmdbuilder.go delete mode 100644 pkg/ytdl/cmdopts.go delete mode 100644 pkg/ytdl/error.go diff --git a/app/controllers/download.go b/app/controllers/download.go index 7dcd793..56d6a55 100644 --- a/app/controllers/download.go +++ b/app/controllers/download.go @@ -5,7 +5,6 @@ import ( "io" "net/http" "net/url" - "sync" "github.com/go-chi/chi/v5" "github.com/spf13/viper" @@ -109,6 +108,10 @@ func (c *DownloadController) ListDownloadLinks(w http.ResponseWriter, r *http.Re }, layout...) } +var ( + BUF_LEN = 1024 +) + func (c *DownloadController) ProxyDownload(w http.ResponseWriter, r *http.Request) { videoUrl, ok := c.getUrlParam(r) if !ok { @@ -159,33 +162,19 @@ func (c *DownloadController) ProxyDownload(w http.ResponseWriter, r *http.Reques index = -1 } - pread, pwrite := io.Pipe() - var wg sync.WaitGroup - wg.Add(2) + read, write := io.Pipe() go func() { - defer wg.Done() - - if err := c.ytdl.Download(pwrite, videoUrl, format.FormatID, index); err != nil { - slog.Error("Failed to download", slog.String("error", err.Error())) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - pwrite.CloseWithError(err) - return - } else { - pwrite.Close() - } - }() - - go func() { - defer wg.Done() - - _, err = io.Copy(w, pread) + _, err := io.Copy(w, read) if err != nil { slog.Error("Failed to copy", slog.String("error", err.Error())) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return } }() - wg.Wait() + if err := c.ytdl.Download(write, videoUrl, format.FormatID, index); err != nil { + slog.Error("Failed to download", slog.String("error", err.Error())) + http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) + } + + write.Close() } diff --git a/config/config.go b/config/config.go index d945278..77bacb2 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "os" + "path" "strings" "time" @@ -10,11 +11,11 @@ import ( ) type Config struct { - Env string `mapstructure:"env"` - BinaryPath string `mapstructure:"binaryPath"` - HTTP ConfigHTTP `mapstructure:"http"` - Cache ConfigCache `mapstructure:"cache"` - Cookies ConfigCookies `mapstructure:"cookies"` + Env string `mapstructure:"env"` + Ytdlp ConfigYtdlp `mapstructure:"ytdlp"` + HTTP ConfigHTTP `mapstructure:"http"` + Cache ConfigCache `mapstructure:"cache"` + Cookies ConfigCookies `mapstructure:"cookies"` } func (c *Config) IsProduction() bool { @@ -29,6 +30,10 @@ func (c *Config) IsStaging() bool { return c.Env == "Staging" } +type ConfigYtdlp struct { + BinaryPath string `mapstructure:"binaryPath"` +} + type ConfigHTTP struct { Port int `mapstructure:"port"` Listen string `mapstructure:"listen"` @@ -56,8 +61,8 @@ type ConfigCookiesFromBrowser struct { func DefaultConfig() *Config { return &Config{ - Env: "Production", - BinaryPath: "yt-dlp", + Env: "Production", + Ytdlp: ConfigYtdlp{BinaryPath: "yt-dlp"}, HTTP: ConfigHTTP{ Port: 8080, Listen: "127.0.0.1", @@ -76,46 +81,46 @@ func DefaultConfig() *Config { } func LoadConfig(paths ...string) (*Config, error) { - v := viper.New() + viper.SetEnvPrefix("YTDL") + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.AutomaticEnv() - v.SetEnvPrefix("YTDL") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - v.AutomaticEnv() - - v.SetConfigName("config") - v.SetConfigType("yaml") + viper.SetConfigName("config") + viper.SetConfigType("yaml") if len(paths) > 0 { - for _, path := range paths { - v.AddConfigPath(path) + for _, p := range paths { + viper.AddConfigPath(path.Dir(p)) } } else { envDir := os.Getenv("YTDL_CONFIGDIR") if envDir != "" { - v.AddConfigPath(envDir) + viper.AddConfigPath(envDir) } - v.AddConfigPath(".") + viper.AddConfigPath(".") homeDir, err := os.UserHomeDir() if err == nil { - v.AddConfigPath(homeDir + "/.config/ytdl-web") + viper.AddConfigPath(homeDir + "/.config/ytdl-web") } - v.AddConfigPath(xdg.ConfigHome + "/ytdl-web") + viper.AddConfigPath(xdg.ConfigHome + "/ytdl-web") for _, dir := range xdg.ConfigDirs { - v.AddConfigPath(dir + "/ytdl-web") + viper.AddConfigPath(dir + "/ytdl-web") } } - if err := v.ReadInConfig(); err != nil { + if err := viper.ReadInConfig(); err != nil { if _, ok := err.(viper.ConfigFileNotFoundError); !ok { return nil, err } } + viper.Debug() + config := DefaultConfig() - if err := v.Unmarshal(config); err != nil { + if err := viper.Unmarshal(config); err != nil { return nil, err } diff --git a/pkg/ytdl/cmdbuilder.go b/pkg/ytdl/cmdbuilder.go deleted file mode 100644 index c3eea43..0000000 --- a/pkg/ytdl/cmdbuilder.go +++ /dev/null @@ -1,56 +0,0 @@ -package ytdl - -import ( - "bytes" - "encoding/json" - "os/exec" - - "go.fifitido.net/ytdl-web/pkg/ytdl/metadata" -) - -func Exec(binary, url string, options ...Option) error { - opts := Options{ - args: []string{url}, - stdin: nil, - stdout: nil, - stderr: nil, - output: nil, - } - for _, opt := range options { - if err := opt(&opts); err != nil { - return err - } - } - - cmd := exec.Command(binary, opts.args...) - - if opts.stdin != nil { - cmd.Stdin = opts.stdin - } - - if opts.stdout != nil { - cmd.Stdout = opts.stdout - } - - var stderr = new(bytes.Buffer) - if opts.stderr != nil { - cmd.Stderr = stderr - } - - if err := cmd.Run(); err != nil { - return &Error{ - stderr: stderr.String(), - child: err, - } - } - - buf, bufOk := opts.stdout.(*bytes.Buffer) - meta, metaOk := opts.output.(*metadata.Metadata) - if bufOk && metaOk { - if err := json.Unmarshal(buf.Bytes(), meta); err != nil { - return err - } - } - - return nil -} diff --git a/pkg/ytdl/cmdopts.go b/pkg/ytdl/cmdopts.go deleted file mode 100644 index ec15215..0000000 --- a/pkg/ytdl/cmdopts.go +++ /dev/null @@ -1,99 +0,0 @@ -package ytdl - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - "strings" - - "go.fifitido.net/ytdl-web/pkg/ytdl/metadata" -) - -type Options struct { - args []string - stdin io.Reader - stdout io.Writer - stderr io.Writer - output interface{} -} - -type Option func(*Options) error - -func WithFormat(format string) Option { - return func(opts *Options) error { - opts.args = append(opts.args, "--format", format) - return nil - } -} - -func WithCookieFile(cookieFile string) Option { - return func(opts *Options) error { - opts.args = append(opts.args, "--cookies", cookieFile) - return nil - } -} - -func WithDumpJson(meta *metadata.Metadata) Option { - return func(opts *Options) error { - opts.args = append(opts.args, "--dump-single-json") - opts.stdout = new(bytes.Buffer) - opts.output = meta - return nil - } -} - -func WithLoadJson(metadata *metadata.Metadata) Option { - return func(opts *Options) error { - opts.args = append(opts.args, "--load-info-json", "-") - - json, err := json.Marshal(metadata) - if err != nil { - return err - } - - opts.stdin = bytes.NewBuffer(json) - return nil - } -} - -func WithBrowserCookies(browser, keyring, profile, container string) Option { - return func(opts *Options) error { - var sb strings.Builder - sb.WriteString(browser) - - if keyring != "" { - sb.WriteByte('+') - sb.WriteString(keyring) - } - - if profile != "" { - sb.WriteByte(':') - sb.WriteString(profile) - } - - if container != "" { - sb.WriteByte(':') - sb.WriteByte(':') - sb.WriteString(container) - } - - opts.args = append(opts.args, "--cookies-from-browser", sb.String()) - return nil - } -} - -func WithStreamOutput(output io.Writer) Option { - return func(opts *Options) error { - opts.args = append(opts.args, "--output", "-") - opts.stdout = output - return nil - } -} - -func WithPlaylistIndex(index int) Option { - return func(opts *Options) error { - opts.args = append(opts.args, "--playlist-items", fmt.Sprint(index+1)) - return nil - } -} diff --git a/pkg/ytdl/error.go b/pkg/ytdl/error.go deleted file mode 100644 index 74ba066..0000000 --- a/pkg/ytdl/error.go +++ /dev/null @@ -1,14 +0,0 @@ -package ytdl - -type Error struct { - stderr string - child error -} - -func (e *Error) Error() string { - return e.child.Error() -} - -func (e *Error) Stderr() string { - return e.stderr -} diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index 63edd63..03b8c47 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -2,7 +2,8 @@ package ytdl import ( "bytes" - "errors" + "encoding/json" + "fmt" "io" "os/exec" "strings" @@ -28,7 +29,7 @@ type ytdlImpl struct { func NewYtdl(cfg *config.Config, logger *slog.Logger, cache cache.MetadataCache) Ytdl { cmd := exec.Command( - cfg.BinaryPath, + cfg.Ytdlp.BinaryPath, "--version", ) var out bytes.Buffer @@ -43,28 +44,44 @@ func NewYtdl(cfg *config.Config, logger *slog.Logger, cache cache.MetadataCache) } } -func (y *ytdlImpl) baseOptions(url string) []Option { - options := []Option{} +func buildBrowserCookieString(browser, keyring, profile, container string) string { + var sb strings.Builder + sb.WriteString(browser) - metadata, err := y.cache.Get(url) - if err == nil { - options = append(options, WithLoadJson(metadata)) + if keyring != "" { + sb.WriteByte('+') + sb.WriteString(keyring) } + if profile != "" { + sb.WriteByte(':') + sb.WriteString(profile) + } + + if container != "" { + sb.WriteByte(':') + sb.WriteByte(':') + sb.WriteString(container) + } + + return sb.String() +} + +func (y *ytdlImpl) appendCookieArgs(args []string) []string { if y.cfg.Cookies.Enabled { if y.cfg.Cookies.FromBrowser.Browser != "" { - options = append(options, WithBrowserCookies( + args = append(args, "--cookies-from-browser", buildBrowserCookieString( y.cfg.Cookies.FromBrowser.Browser, y.cfg.Cookies.FromBrowser.Keyring, y.cfg.Cookies.FromBrowser.Profile, y.cfg.Cookies.FromBrowser.Container, )) } else { - options = append(options, WithCookieFile(y.cfg.Cookies.FilePath)) + args = append(args, "--cookies", y.cfg.Cookies.FilePath) } } - return options + return args } func (y *ytdlImpl) Version() string { @@ -78,27 +95,32 @@ func (y *ytdlImpl) GetMetadata(url string) (*metadata.Metadata, error) { return meta, nil } - meta = &metadata.Metadata{} - options := append( - y.baseOptions(url), - WithDumpJson(meta), - ) + args := []string{ + url, + "--dump-single-json", + } - if err := Exec(y.cfg.BinaryPath, url, options...); err != nil { + args = y.appendCookieArgs(args) + + cmd := exec.Command(y.cfg.Ytdlp.BinaryPath, args...) + + out, err := cmd.Output() + if err != nil { attrs := []any{ slog.String("url", url), slog.String("error", err.Error()), } - var ytdlErr *Error - if ok := errors.As(err, &ytdlErr); ok { - attrs = append(attrs, slog.String("stderr", ytdlErr.Stderr())) - } - y.logger.Error("failed to get metadata", attrs...) return nil, err } + meta = &metadata.Metadata{} + if err := json.Unmarshal(out, meta); err != nil { + y.logger.Error("failed to unmarshal metadata", slog.String("url", url), slog.String("error", err.Error())) + return nil, err + } + if err := y.cache.Set(url, meta, y.cfg.Cache.TTL); err != nil { y.logger.Warn("failed to cache metadata", slog.String("url", url), slog.String("error", err.Error())) } @@ -108,17 +130,37 @@ func (y *ytdlImpl) GetMetadata(url string) (*metadata.Metadata, error) { // Download implements Ytdl func (y *ytdlImpl) Download(w io.Writer, url, format string, index int) error { - options := append( - y.baseOptions(url), - WithFormat(format), - WithStreamOutput(w), - ) - - if index >= 0 { - options = append(options, WithPlaylistIndex(index)) + args := []string{ + url, + "--format", format, + "--output", "-", } - if err := Exec(y.cfg.BinaryPath, url, options...); err != nil { + if index >= 0 { + args = append(args, "--playlist-ites", fmt.Sprint(index+1)) + } + + args = y.appendCookieArgs(args) + + metadata, err := y.cache.Get(url) + if err == nil { + args = append(args, "--load-info-json", "-") + } + + cmd := exec.Command(y.cfg.Ytdlp.BinaryPath, args...) + cmd.Stdout = w + + if err == nil { + json, err := json.Marshal(metadata) + if err != nil { + return err + } + + cmd.Stdin = bytes.NewReader(json) + } + + if err := cmd.Run(); err != nil { + y.logger.Error("failed to download", slog.String("url", url), slog.String("error", err.Error())) return err }