From 532408fc09a68b34e0d19e9b271d1c4eafdcd45b Mon Sep 17 00:00:00 2001 From: Evan Fiordeliso Date: Tue, 23 May 2023 18:44:05 -0400 Subject: [PATCH] Command builder with caching --- .gitignore | 4 +- cmd/root.go | 62 ++++-- config.example.yaml | 44 ++++ config.example.yml | 4 - config/config.go | 108 +++++++++ go.mod | 11 +- go.sum | 42 +++- web/formats.go | 14 +- web/routes.go | 13 +- web/serve.go | 16 +- web/views.go | 4 +- ytdl/cache/cache.go | 49 ++++ ytdl/cmdbuilder.go | 52 +++++ ytdl/cmdopts.go | 91 ++++++++ ytdl/data.go | 446 ------------------------------------- ytdl/meta.go | 36 --- ytdl/metadata/chapter.go | 11 + ytdl/metadata/comment.go | 43 ++++ ytdl/metadata/format.go | 160 +++++++++++++ ytdl/metadata/fragment.go | 12 + ytdl/metadata/hls.go | 13 ++ ytdl/metadata/metadata.go | 178 +++++++++++++++ ytdl/metadata/subtitle.go | 18 ++ ytdl/metadata/thumbnail.go | 18 ++ ytdl/stream.go | 26 --- ytdl/ytdl.go | 87 ++++++++ 26 files changed, 1004 insertions(+), 558 deletions(-) create mode 100644 config.example.yaml delete mode 100644 config.example.yml create mode 100644 config/config.go create mode 100644 ytdl/cache/cache.go create mode 100644 ytdl/cmdbuilder.go create mode 100644 ytdl/cmdopts.go delete mode 100644 ytdl/data.go delete mode 100644 ytdl/meta.go create mode 100644 ytdl/metadata/chapter.go create mode 100644 ytdl/metadata/comment.go create mode 100644 ytdl/metadata/format.go create mode 100644 ytdl/metadata/fragment.go create mode 100644 ytdl/metadata/hls.go create mode 100644 ytdl/metadata/metadata.go create mode 100644 ytdl/metadata/subtitle.go create mode 100644 ytdl/metadata/thumbnail.go delete mode 100644 ytdl/stream.go create mode 100644 ytdl/ytdl.go diff --git a/.gitignore b/.gitignore index 1ef0652..f7d50ba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /out/ /tmp/ .deploy -dist/ \ No newline at end of file +dist/ +config.yaml +cookies.txt \ No newline at end of file diff --git a/cmd/root.go b/cmd/root.go index 40826e1..08cf0b5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -4,18 +4,18 @@ Copyright © 2023 Evan Fiordeliso package cmd import ( - "fmt" "os" - "github.com/adrg/xdg" "github.com/spf13/cobra" "github.com/spf13/viper" + "go.fifitido.net/ytdl-web/config" "go.fifitido.net/ytdl-web/web" "golang.org/x/exp/slog" ) var ( cfgFile string + cfg *config.Config rootCmd = &cobra.Command{ Use: "ytdl-web", @@ -25,7 +25,7 @@ var ( A web application that grabs the links to videos from over a thousand websites using the yt-dlp project under the hood.`, Run: func(cmd *cobra.Command, args []string) { - if err := web.Serve(); err != nil { + if err := web.Serve(cfg); err != nil { slog.Error("Error when serving website", slog.String("error", err.Error())) } }, @@ -48,35 +48,51 @@ func init() { rootCmd.PersistentFlags().StringP("listen", "l", "127.0.0.1", "address to listen on") rootCmd.PersistentFlags().StringP("base-path", "b", "", "the base path, used when behind reverse proxy") rootCmd.PersistentFlags().StringP("ytdlp-path", "y", "yt-dlp", "the path to the yt-dlp binary, used when it is not in $PATH") + rootCmd.PersistentFlags().BoolP("cookies-enabled", "C", false, "whether cookies are enabled") + rootCmd.PersistentFlags().StringP("cookies", "c", "", "the path to the cookies file") // trunk-ignore-begin(golangci-lint/errcheck): Ignoring errors - viper.BindPFlag("port", rootCmd.PersistentFlags().Lookup("port")) - viper.BindPFlag("listen", rootCmd.PersistentFlags().Lookup("listen")) - viper.BindPFlag("base_path", rootCmd.PersistentFlags().Lookup("base-path")) - viper.BindPFlag("ytdlp_path", rootCmd.PersistentFlags().Lookup("ytdlp-path")) + viper.BindPFlag("http.port", rootCmd.PersistentFlags().Lookup("port")) + viper.BindPFlag("http.listen", rootCmd.PersistentFlags().Lookup("listen")) + viper.BindPFlag("http.basePath", rootCmd.PersistentFlags().Lookup("base-path")) + viper.BindPFlag("ytdlp.binaryPath", rootCmd.PersistentFlags().Lookup("ytdlp-path")) + viper.BindPFlag("cookies.enabled", rootCmd.PersistentFlags().Lookup("cookies-enabled")) + viper.BindPFlag("cookies.filePath", rootCmd.PersistentFlags().Lookup("cookies")) // trunk-ignore-end(golangci-lint/errcheck) - - viper.SetDefault("port", 8080) - viper.SetDefault("listen", "127.0.0.1") - viper.SetDefault("base_path", "") - viper.SetDefault("ytdlp_path", "yt-dlp") } func initConfig() { - slog.Info("Initializing configuration") - + var err error if cfgFile != "" { - viper.SetConfigFile(cfgFile) + cfg, err = config.LoadConfig(cfgFile) } else { - viper.AddConfigPath(xdg.ConfigHome) - viper.SetConfigType("yml") - viper.SetConfigName("config") + cfg, err = config.LoadConfig() } - viper.SetEnvPrefix("ytdl") - viper.AutomaticEnv() - - if err := viper.ReadInConfig(); err == nil { - fmt.Println("Using config file:", viper.ConfigFileUsed()) + if err != nil { + slog.Error("Error loading configuration", slog.String("error", err.Error())) + os.Exit(1) } + + initLogging() + + slog.Info("Configuration loaded") +} + +func initLogging() { + var handler slog.Handler + + if cfg.IsProduction() { + handler = slog.HandlerOptions{ + AddSource: true, + Level: slog.LevelWarn, + }.NewJSONHandler(os.Stdout) + } else { + handler = slog.HandlerOptions{ + AddSource: true, + Level: slog.LevelDebug, + }.NewTextHandler(os.Stdout) + } + + slog.SetDefault(slog.New(handler)) } diff --git a/config.example.yaml b/config.example.yaml new file mode 100644 index 0000000..1546a65 --- /dev/null +++ b/config.example.yaml @@ -0,0 +1,44 @@ +# The server environment +# For dev environments use Development +# For prod environments use Production +# For staging envronments use Staging +env: Production +http: + # The port to listen on + port: 8080 + # The address to listen on + # For local only access use 127.0.0.1 + # For public access use 0.0.0.0 + listen: 0.0.0.0 + # The base path of the application, useful for reverse proxies + basePath: "" + + # A list of proxy servers to trust for security purposes + # Only needed when accessing app behind a proxy + trustedProxies: [] +ytdlp: + # The path to the yt-dlp binary, if it is already in your $PATH just yt-dlp will work. + binaryPath: yt-dlp +cookies: + # Whether to use cookies when fetching the video metadata + enabled: false + + # The path to the netscape formatted cookies file + # See: https://www.reddit.com/r/youtubedl/wiki/cookies/ for details. + filePath: ~/.cookies + + # Settings for using cookies from a browser's cookies store + fromBrowser: + # The name of the browser to load cookies from. + # Currently supported browsers are: brave, chrome, chromium, edge, firefox, opera, safari, vivaldi. + browser: firefox + + # The keyring used for decrypting Chromium cookies on Linux + # Currently supported keyrings are: basictext, gnomekeyring, kwallet + keyring: basictext + + # The profile to load cookies from (Firefox) + profile: default + + # The container to load cookies from (Firefox) + container: none diff --git a/config.example.yml b/config.example.yml deleted file mode 100644 index 2b669c4..0000000 --- a/config.example.yml +++ /dev/null @@ -1,4 +0,0 @@ -port: 8080 -listen: 0.0.0.0 -base_path: "" -ytdlp_path: yt-dlp diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..0098ab7 --- /dev/null +++ b/config/config.go @@ -0,0 +1,108 @@ +package config + +import ( + "time" + + "github.com/spf13/viper" +) + +type Config struct { + Env string `mapstructure:"env"` + HTTP ConfigHTTP `mapstructure:"http"` + Ytdlp ConfigYtdlp `mapstructure:"ytdlp"` + Cookies ConfigCookies `mapstructure:"cookies"` +} + +func (c *Config) IsProduction() bool { + return c.Env == "Production" +} + +func (c *Config) IsDevelopment() bool { + return c.Env == "Development" +} + +func (c *Config) IsStaging() bool { + return c.Env == "Staging" +} + +type ConfigHTTP struct { + Port int `mapstructure:"port"` + Listen string `mapstructure:"listen"` + BasePath string `mapstructure:"basePath"` + TrustedProxies []string `mapstructure:"trustedProxies"` +} + +type ConfigYtdlp struct { + BinaryPath string `mapstructure:"binaryPath"` + Cache ConfigYtdlpCache `mapstructure:"cache"` +} + +type ConfigYtdlpCache struct { + TTL time.Duration `mapstructure:"ttl"` +} + +type ConfigCookies struct { + Enabled bool `mapstructure:"enabled"` + FilePath string `mapstructure:"filePath"` + FromBrowser *ConfigCookiesFromBrowser `mapstructure:"fromBrowser"` +} + +type ConfigCookiesFromBrowser struct { + Browser string `mapstructure:"browser"` + Keyring string `mapstructure:"keyring"` + Profile string `mapstructure:"profile"` + Container string `mapstructure:"container"` +} + +func DefaultConfig() *Config { + return &Config{ + HTTP: ConfigHTTP{ + Port: 8080, + Listen: "127.0.0.1", + BasePath: "/", + }, + Ytdlp: ConfigYtdlp{ + BinaryPath: "yt-dlp", + Cache: ConfigYtdlpCache{ + TTL: time.Hour, + }, + }, + Cookies: ConfigCookies{ + Enabled: false, + FilePath: "/tmp/ytdl-web.cookies", + FromBrowser: nil, + }, + } +} + +func LoadConfig(paths ...string) (*Config, error) { + v := viper.New() + + v.SetEnvPrefix("YTDL") + v.AutomaticEnv() + + v.SetConfigName("config") + v.SetConfigType("yaml") + + if len(paths) > 0 { + for _, path := range paths { + v.AddConfigPath(path) + } + } else { + v.AddConfigPath(".") + v.AddConfigPath("/etc/ytdl-web") + v.AddConfigPath("$HOME/.config/ytdl-web") + v.AddConfigPath("$XDG_CONFIG_HOME/ytdl-web") + } + + if err := v.ReadInConfig(); err != nil { + return nil, err + } + + config := DefaultConfig() + if err := v.Unmarshal(config); err != nil { + return nil, err + } + + return config, nil +} diff --git a/go.mod b/go.mod index c5aa9b7..ee073fc 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module go.fifitido.net/ytdl-web go 1.20 require ( - github.com/adrg/xdg v0.4.0 + github.com/dgraph-io/badger/v2 v2.2007.4 github.com/gofiber/fiber/v2 v2.43.0 github.com/gofiber/template v1.8.0 github.com/htfy96/reformism v0.0.0-20160819020323-e5bfca398e73 @@ -16,7 +16,13 @@ require ( require ( github.com/andybalholm/brotli v1.0.5 // indirect + github.com/cespare/xxhash v1.1.0 // indirect + github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de // indirect + github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.3 // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -28,6 +34,7 @@ require ( github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/pelletier/go-toml v1.9.4 // indirect github.com/philhofer/fwd v1.1.2 // indirect + github.com/pkg/errors v0.8.1 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 // indirect github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee // indirect @@ -40,8 +47,10 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.45.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/ini.v1 v1.66.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 9788d5d..c9006dd 100644 --- a/go.sum +++ b/go.sum @@ -53,9 +53,8 @@ github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3/go.mod h1:T+2WLyt7VH6Lp0TRxQrUYEs64nRc83wkMQrfeIQKduM= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= -github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -65,6 +64,7 @@ github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/ github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -77,6 +77,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/cbroglie/mustache v1.4.0/go.mod h1:SS1FTIghy0sjse4DUVGV1k/40B1qE1XkD9DtDsHo9iM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -96,14 +97,26 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= +github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -120,6 +133,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -169,7 +183,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -185,6 +201,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -261,6 +278,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= @@ -275,6 +293,7 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -317,12 +336,14 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWb github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -345,6 +366,7 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= @@ -359,18 +381,26 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.10.0 h1:mXH0UwHS4D2HwWZa75im4xIQynLfblmWV7qcWpfv0yk= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -390,6 +420,7 @@ github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOM github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.38.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= @@ -397,6 +428,7 @@ github.com/valyala/fasthttp v1.45.0 h1:zPkkzpIn8tdHZUrVa6PzYd0i5verqiPSkgTd3bSUc github.com/valyala/fasthttp v1.45.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yosssi/ace v0.0.5/go.mod h1:ALfIzm2vT7t5ZE7uoIZqF3TQ7SAOyupFZnkrF5id+K0= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -421,6 +453,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -522,6 +555,7 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -558,6 +592,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -567,6 +602,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -617,7 +653,6 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -855,6 +890,7 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/web/formats.go b/web/formats.go index a2e6398..b1d8342 100644 --- a/web/formats.go +++ b/web/formats.go @@ -2,22 +2,22 @@ package web import ( "github.com/samber/lo" - "go.fifitido.net/ytdl-web/ytdl" + "go.fifitido.net/ytdl-web/ytdl/metadata" ) type Video struct { - Meta ytdl.Metadata - Formats []ytdl.Format + Meta *metadata.Metadata + Formats []metadata.Format } -func GetVideos(meta ytdl.Metadata) []Video { +func GetVideos(meta *metadata.Metadata) []Video { if meta.Type == "playlist" { - return lo.Map(meta.Entries, func(video ytdl.Metadata, _ int) Video { - return GetVideos(video)[0] + return lo.Map(meta.Entries, func(video metadata.Metadata, _ int) Video { + return GetVideos(&video)[0] }) } - formats := lo.Filter(meta.Formats, func(item ytdl.Format, _ int) bool { + formats := lo.Filter(meta.Formats, func(item metadata.Format, _ int) bool { return item.ACodec != "none" && item.VCodec != "none" && item.Protocol != "m3u8_native" }) diff --git a/web/routes.go b/web/routes.go index 8a770a7..7f7f19e 100644 --- a/web/routes.go +++ b/web/routes.go @@ -10,10 +10,13 @@ import ( "github.com/sujit-baniya/flash" "go.fifitido.net/ytdl-web/version" "go.fifitido.net/ytdl-web/ytdl" + "go.fifitido.net/ytdl-web/ytdl/metadata" "golang.org/x/exp/slog" ) -type routes struct{} +type routes struct { + ytdl ytdl.Ytdl +} func (r *routes) Register(app *fiber.App) { app.Get("/", r.IndexHandler) @@ -48,7 +51,7 @@ func (r *routes) DownloadHandler(c *fiber.Ctx) error { }).Redirect("/") } - meta, err := ytdl.GetMetadata(url) + meta, err := r.ytdl.GetMetadata(url) if err != nil { return flash.WithError(c, fiber.Map{ "error": true, @@ -83,13 +86,13 @@ func (r *routes) DownloadProxyHandler(c *fiber.Ctx) error { return fiber.ErrBadRequest } - meta, err := ytdl.GetMetadata(url) + meta, err := r.ytdl.GetMetadata(url) if err != nil { slog.Error("Failed to get metadata", slog.String("error", err.Error())) return fiber.ErrInternalServerError } - format, ok := lo.Find(meta.Formats, func(format ytdl.Format) bool { + format, ok := lo.Find(meta.Formats, func(format metadata.Format) bool { return format.FormatID == formatId }) if !ok { @@ -103,5 +106,5 @@ func (r *routes) DownloadProxyHandler(c *fiber.Ctx) error { c.Set("Content-Length", fmt.Sprint(*format.FilesizeApprox)) } - return ytdl.Stream(c.Response().BodyWriter(), url, format) + return r.ytdl.Download(c.Response().BodyWriter(), url, format.FormatID) } diff --git a/web/serve.go b/web/serve.go index ffe18f3..4663eb8 100644 --- a/web/serve.go +++ b/web/serve.go @@ -4,17 +4,25 @@ import ( "fmt" "github.com/gofiber/fiber/v2" - "github.com/spf13/viper" + "go.fifitido.net/ytdl-web/config" + "golang.org/x/exp/slog" ) -func Serve() error { +func Serve(cfg *config.Config) error { engine := ViewsEngine() - app := fiber.New(fiber.Config{Views: engine}) + app := fiber.New(fiber.Config{ + Views: engine, + EnableTrustedProxyCheck: true, + TrustedProxies: cfg.HTTP.TrustedProxies, + DisableStartupMessage: true, + }) routes := &routes{} routes.Register(app) - listenAddr := fmt.Sprintf("%s:%d", viper.GetString("listen"), viper.GetInt("port")) + listenAddr := fmt.Sprintf("%s:%d", cfg.HTTP.Listen, cfg.HTTP.Port) + + slog.Info("Starting HTTP server", slog.String("host", cfg.HTTP.Listen), slog.Int("port", cfg.HTTP.Port)) return app.Listen(listenAddr) } diff --git a/web/views.go b/web/views.go index 26c6d4f..077932f 100644 --- a/web/views.go +++ b/web/views.go @@ -9,7 +9,7 @@ import ( "github.com/gofiber/template/html" "github.com/htfy96/reformism" - "go.fifitido.net/ytdl-web/ytdl" + "go.fifitido.net/ytdl-web/ytdl/metadata" ) //go:embed views/* @@ -37,7 +37,7 @@ func ViewsEngine() *html.Engine { }, ) engine.AddFunc( - "downloadContext", func(meta ytdl.Metadata, url, basePath string, format ytdl.Format) map[string]any { + "downloadContext", func(meta metadata.Metadata, url, basePath string, format metadata.Format) map[string]any { return map[string]any{ "Meta": meta, "Url": url, diff --git a/ytdl/cache/cache.go b/ytdl/cache/cache.go new file mode 100644 index 0000000..ff125c7 --- /dev/null +++ b/ytdl/cache/cache.go @@ -0,0 +1,49 @@ +package cache + +import ( + "encoding/json" + "time" + + "github.com/dgraph-io/badger/v2" + "go.fifitido.net/ytdl-web/ytdl/metadata" +) + +type MetadataCache interface { + Get(key string) (*metadata.Metadata, error) + Set(key string, value *metadata.Metadata, ttl time.Duration) error +} + +type DefaultMetadataCache struct { + db *badger.DB +} + +func NewDefaultMetadataCache(db *badger.DB) *DefaultMetadataCache { + return &DefaultMetadataCache{ + db: db, + } +} +func (c *DefaultMetadataCache) Get(key string) (*metadata.Metadata, error) { + value := &metadata.Metadata{} + err := c.db.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(key)) + if err != nil { + return err + } + + return item.Value(func(val []byte) error { + return json.Unmarshal(val, value) + }) + }) + return value, err +} + +func (c *DefaultMetadataCache) Set(key string, value *metadata.Metadata, ttl time.Duration) error { + return c.db.Update(func(txn *badger.Txn) error { + data, err := json.Marshal(value) + if err != nil { + return err + } + e := badger.NewEntry([]byte(key), data).WithTTL(ttl) + return txn.SetEntry(e) + }) +} diff --git a/ytdl/cmdbuilder.go b/ytdl/cmdbuilder.go new file mode 100644 index 0000000..80d4dc5 --- /dev/null +++ b/ytdl/cmdbuilder.go @@ -0,0 +1,52 @@ +package ytdl + +import ( + "bytes" + "encoding/json" + "os/exec" + + "go.fifitido.net/ytdl-web/ytdl/metadata" +) + +func Exec(url string, options ...Option) error { + opts := Options{ + args: []string{url}, + stdin: nil, + stdout: nil, + stderr: nil, + output: nil, + } + for _, opt := range options { + if err := opt(&opts); err != nil { + return err + } + } + + cmd := exec.Command("youtube-dl", opts.args...) + + if opts.stdin != nil { + cmd.Stdin = opts.stdin + } + + if opts.stdout != nil { + cmd.Stdout = opts.stdout + } + + if opts.stderr != nil { + cmd.Stderr = opts.stderr + } + + if err := cmd.Run(); err != nil { + return err + } + + buf, bufOk := opts.stdout.(*bytes.Buffer) + meta, metaOk := opts.output.(*metadata.Metadata) + if bufOk && metaOk { + if err := json.Unmarshal(buf.Bytes(), meta); err != nil { + return err + } + } + + return nil +} diff --git a/ytdl/cmdopts.go b/ytdl/cmdopts.go new file mode 100644 index 0000000..cf3c25d --- /dev/null +++ b/ytdl/cmdopts.go @@ -0,0 +1,91 @@ +package ytdl + +import ( + "bytes" + "encoding/json" + "io" + "strings" + + "go.fifitido.net/ytdl-web/ytdl/metadata" +) + +type Options struct { + args []string + stdin io.Reader + stdout io.Writer + stderr io.Writer + output interface{} +} + +type Option func(*Options) error + +func WithFormat(format string) Option { + return func(opts *Options) error { + opts.args = append(opts.args, "--format", format) + return nil + } +} + +func WithCookieFile(cookieFile string) Option { + return func(opts *Options) error { + opts.args = append(opts.args, "--cookies", cookieFile) + return nil + } +} + +func WithDumpJson(meta *metadata.Metadata) Option { + return func(opts *Options) error { + opts.args = append(opts.args, "--dump-single-json") + opts.stdout = new(bytes.Buffer) + opts.output = meta + return nil + } +} + +func WithLoadJson(metadata *metadata.Metadata) Option { + return func(opts *Options) error { + opts.args = append(opts.args, "--load-single-json", "-") + + json, err := json.Marshal(metadata) + if err != nil { + return err + } + + opts.stdin = bytes.NewBuffer(json) + return nil + } +} + +func WithBrowserCookies(browser, keyring, profile, container string) Option { + return func(opts *Options) error { + var sb strings.Builder + sb.WriteString(browser) + + if keyring != "" { + sb.WriteByte('+') + sb.WriteString(keyring) + } + + if profile != "" { + sb.WriteByte(':') + sb.WriteString(profile) + } + + if container != "" { + sb.WriteByte(':') + sb.WriteByte(':') + sb.WriteString(container) + } + + opts.args = append(opts.args, "--cookies-from-browser", sb.String()) + return nil + } +} + +func WithStreamOutput(output io.Writer) Option { + return func(opts *Options) error { + opts.args = append(opts.args, "--output", "-") + opts.stdout = output + return nil + } +} diff --git a/ytdl/data.go b/ytdl/data.go deleted file mode 100644 index 05c6775..0000000 --- a/ytdl/data.go +++ /dev/null @@ -1,446 +0,0 @@ -package ytdl - -type Metadata struct { - Type string `json:"_type"` - - // A list of videos in the playlist - Entries []Metadata - - // Video identifier. - ID string `json:"id"` - - // Video title, unescaped. Set to an empty string if video has - // no title as opposed to "None" which signifies that the - // extractor failed to obtain a title - Title *string `json:"title"` - - // A list of dictionaries for each format available, ordered - // from worst to best quality. - Formats []Format `json:"formats"` - - // Final video URL. - Url string `json:"url"` - - // Video filename extension. - Ext string `json:"ext"` - - // The video format, defaults to ext (used for --get-format) - Format string `json:"format"` - - // True if a direct video file was given (must only be set by GenericIE) - Direct *bool `json:"direct"` - - // A secondary title of the video. - AltTitle *string `json:"alt_title"` - - // An alternative identifier for the video, not necessarily - // unique, but available before title. Typically, id is - // something like "4234987", title "Dancing naked mole rats", - // and display_id "dancing-naked-mole-rats" - DisplayID *string `json:"display_id"` - - Thumbnails []Thumbnail `json:"thumbnails"` - - // Full URL to a video thumbnail image. - Thumbnail *string `json:"thumbnail"` - - // Full video description. - Description *string `json:"description"` - - // Full name of the video uploader. - Uploader *string `json:"uploader"` - - // License name the video is licensed under. - License *string `json:"license"` - - // The creator of the video. - Creator *string `json:"creator"` - - // UNIX timestamp of the moment the video was uploaded - Timestamp *float64 `json:"timestamp"` - - // Video upload date in UTC (YYYYMMDD). - // If not explicitly set, calculated from timestamp - UploadDate *string `json:"upload_date"` - - // UNIX timestamp of the moment the video was released. - // If it is not clear whether to use timestamp or this, use the former - ReleaseTimestamp *float64 `json:"release_timestamp"` - - // The date (YYYYMMDD) when the video was released in UTC. - // If not explicitly set, calculated from release_timestamp - ReleaseDate *string `json:"release_date"` - - // UNIX timestamp of the moment the video was last modified. - ModifiedTimestamp *float64 `json:"modified_timestamp"` - - // The date (YYYYMMDD) when the video was last modified in UTC. - // If not explicitly set, calculated from modified_timestamp - ModifiedDate *string `json:"modified_date"` - - // Nickname or id of the video uploader. - UploaderId *string `json:"uploader_id"` - - // Full URL to a personal webpage of the video uploader. - UploaderUrl *string `json:"uploader_url"` - - // Full name of the channel the video is uploaded on. - // Note that channel fields may or may not repeat uploader - // fields. This depends on a particular extractor. - Channel *string `json:"channel"` - - // Id of the channel. - ChannelId *string `json:"channel_id"` - - // Full URL to a channel webpage. - ChannelUrl *string `json:"channel_url"` - - // Number of followers of the channel. - ChannelFollowerCount *int `json:"channel_follower_count"` - - // Physical location where the video was filmed. - Location *string `json:"location"` - - // The available subtitles as a dictionary in the format - // {tag: subformats}. "tag" is usually a language code, and - // "subformats" is a list sorted from lower to higher - // preference - Subtitles map[string][]Subtitle `json:"subtitles"` - - // Like 'subtitles'; contains automatically generated - // captions instead of normal subtitles - AutomaticCaptions map[string][]Subtitle `json:"automatic_captions"` - - // Length of the video in seconds, as an integer or float. - Duration *float64 `json:"duration"` - - // How many users have watched the video on the platform. - ViewCount *int64 `json:"view_count"` - - // How many users are currently watching the video on the platform. - ConcurrentViewCount *int64 `json:"concurrent_view_count"` - - // Number of positive ratings of the video - LikeCount *int64 `json:"like_count"` - - // Number of negative ratings of the video - DislikeCount *int64 `json:"dislike_count"` - - // Number of reposts of the video - RepostCount *int64 `json:"repost_count"` - - // Average rating give by users, the scale used depends on the webpage - AverageRating *float64 `json:"average_rating"` - - // Number of comments on the video - CommentCount *int64 `json:"comment_count"` - - // A list of comments - Comments []Comment `json:"comments"` - - // Age restriction for the video, as an integer (years) - AgeLimit *int `json:"age_limit"` - - // The URL to the video webpage, if given to yt-dlp it - // should allow to get the same result again. (It will be set - // by YoutubeDL if it's missing) - WebpageUrl *string `json:"webpage_url"` - - // A list of categories that the video falls in - Categories []string `json:"categories"` - - // A list of tags assigned to the video - Tags []string `json:"tags"` - - // A list of the video cast - Cast []string `json:"cast"` - - // Whether this video is a live stream that goes on instead of a fixed-length video. - IsLive *bool `json:"is_live"` - - // Whether this video was originally a live stream. - WasLive *bool `json:"was_live"` - - // None (=unknown), 'is_live', 'is_upcoming', 'was_live', 'not_live', - // or 'post_live' (was live, but VOD is not yet processed) - // If absent, automatically set from is_live, was_live - LiveStatus *string `json:"live_status"` - - // Time in seconds where the reproduction should start, as - // specified in the URL. - StartTime *float64 `json:"start_time"` - - // Time in seconds where the reproduction should end, as - // specified in the URL. - EndTime *float64 `json:"end_time"` - - Chapters []Chapter `json:"chapters"` -} - -type Format struct { - // The mandatory URL representing the media: - // for plain file media - HTTP URL of this file, - // for RTMP - RTMP URL, - // for HLS - URL of the M3U8 media playlist, - // for HDS - URL of the F4M manifest, - // for DASH - // - HTTP URL to plain file media (in case of - // unfragmented media) - // - URL of the MPD manifest or base URL - // representing the media if MPD manifest - // is parsed from a string (in case of - // fragmented media) - // for MSS - URL of the ISM manifest. - Url string `json:"url"` - - // Will be calculated from URL if missing - Ext string `json:"ext"` - - // A human-readable description of the format - // ("mp4 container with h264/opus"). - // Calculated from the format_id, width, height. - // and format_note fields if missing. - Format string `json:"format"` - - // A short description of the format - // ("mp4_h264_opus" or "19"). - // Technically optional, but strongly recommended. - FormatID string `json:"format_id"` - - // Additional info about the format - // ("3D" or "DASH video") - FormatNote string `json:"format_note"` - - // Width of the video, if known - Width int `json:"width"` - - // Height of the video, if known - Height int `json:"height"` - - // Aspect ratio of the video, if known - // Automatically calculated from width and height - AspectRatio float64 `json:"aspect_ratio"` - - // Textual description of width and height - // Automatically calculated from width and height - Resolution string `json:"resolution"` - - // The dynamic range of the video. One of: - // "SDR" (None), "HDR10", "HDR10+, "HDR12", "HLG, "DV" - DynamicRange string `json:"dynamic_range"` - - // Average bitrate of audio and video in KBit/s - Tbr float64 `json:"tbr"` - - // Average audio bitrate in KBit/s - Abr float64 `json:"abr"` - - // Average video bitrate in KBit/s - Vbr float64 `json:"vbr"` - - // Name of the audio codec in use - ACodec string `json:"acodec"` - - // Name of the video codec in use - VCodec string `json:"vcodec"` - - // Number of audio channels - AudioChannels int `json:"audio_channels"` - - // Frame rate - Fps float64 `json:"fps"` - - // Name of the container format - Container string `json:"container"` - - // The number of bytes, if known in advance - Filesize *int `json:"filesize"` - - // An estimate for the number of bytes - FilesizeApprox *int `json:"filesize_approx"` - - // The protocol that will be used for the actual - // download, lower-case. One of "http", "https" or - // one of the protocols defined in downloader.PROTOCOL_MAP - Protocol string `json:"protocol"` - - // Base URL for fragments. Each fragment's path - // value (if present) will be relative to - // this URL. - FragmentBaseUrl *string `json:"fragment_base_url"` - - // A list of fragments of a fragmented media. - // Each fragment entry must contain either an url - // or a path. If an url is present it should be - // considered by a client. Otherwise both path and - // fragment_base_url must be present. - Fragments []Fragment `json:"fragments"` - - // Is a live format that can be downloaded from the start. - IsFromStart bool `json:"is_from_start"` - - // Order number of this format. If this field is - // present and not None, the formats get sorted - // by this field, regardless of all other values. - // -1 for default (order by other properties), - // -2 or smaller for less than default. - // < -1000 to hide the format (if there is - // another one which is strictly better) - Preference *int `json:"preference"` - - // Language code, e.g. "de" or "en-US". - Language string `json:"language"` - - // Is this in the language mentioned in - // the URL? - // 10 if it's what the URL is about, - // -1 for default (don't know), - // -10 otherwise, other values reserved for now. - LanguagePreference int `json:"language_preference"` - - // Order number of the video quality of this - // format, irrespective of the file format. - // -1 for default (order by other properties), - // -2 or smaller for less than default. - Quality float64 `json:"quality"` - - // Order number for this video source - // (quality takes higher priority) - // -1 for default (order by other properties), - // -2 or smaller for less than default. - SourcePreference int `json:"source_preference"` - - // A dictionary of additional HTTP headers to add to the request. - HttpHeaders map[string]string `json:"http_header"` - - // If given and not 1, indicates that the - // video's pixels are not square. - // width : height ratio as float. - StretchedRatio *float64 `json:"stretched_ratio"` - - // The server does not support resuming the (HTTP or RTMP) download. - NoResume bool `json:"no_resume"` - - // The format has DRM and cannot be downloaded. - HasDrm bool `json:"has_drm"` - - // A query string to append to each - // fragment's URL, or to update each existing query string - // with. Only applied by the native HLS/DASH downloaders. - ExtraParamToSegmentUrl string `json:"extra_param_to_segment_url"` - - // A dictionary of HLS AES-128 decryption information - // used by the native HLS downloader to override the - // values in the media playlist when an '#EXT-X-KEY' tag - // is present in the playlist - HlsAes *HlsAes `json:"hls_aes"` -} - -type Fragment struct { - // fragment's URL - Url string `json:"url"` - - // fragment's path relative to fragment_base_url - Path string `json:"path"` - - Duration *float64 `json:"duration"` - Filesize *int `json:"filesize"` -} - -type HlsAes struct { - // The URI from which the key will be downloaded - Uri string `json:"uri"` - - // The key (as hex) used to decrypt fragments. - // If `key` is given, any key URI will be ignored - Key string `json:"key"` - - // The IV (as hex) used to decrypt fragments - Iv string `json:"iv"` -} - -type Thumbnail struct { - // Thumbnail format ID - ID *string `json:"id"` - - Url string `json:"url"` - - // Quality of the image - Preference *int `json:"preference"` - - Width *int `json:"width"` - Height *int `json:"height"` - Filesize *int `json:"filesize"` - - // HTTP headers for the request - HttpHeaders map[string]string `json:"http_headers"` -} - -type Subtitle struct { - // Will be calculated from URL if missing - Ext string `json:"ext"` - - // The subtitles file contents - Data string `json:"data"` - - // A URL pointing to the subtitles file - Url string `json:"url"` - - // Name or description of the subtitles - Name string `json:"name"` - - // A dictionary of additional HTTP headers to add to the request. - HttpHeaders map[string]string `json:"http_headers"` -} - -type Comment struct { - // human-readable name of the comment author - Author *string `json:"author"` - - // user ID of the comment author - AuthorID *string `json:"author_id"` - - // The thumbnail of the comment author - AuthorThumbnail *string `json:"author_thumbnail"` - - // Comment ID - ID *string `json:"id"` - - // Comment as HTML - HTML *string `json:"html"` - - // Plain text of the comment - Text *string `json:"text"` - - // UNIX timestamp of comment - Timestamp *float64 `json:"timestamp"` - - // ID of the comment this one is replying to. - // Set to "root" to indicate that this is a - // comment to the original video. - Parent string `json:"parent"` - - // Number of positive ratings of the comment - LikeCount *int64 `json:"like_count"` - - // Number of negative ratings of the comment - DislikeCount *int64 `json:"dislike_count"` - - // Whether the comment is marked as - // favorite by the video uploader - IsFavorited bool `json:"is_favorited"` - - // Whether the comment is made by - // the video uploader - AuthorIsUploader bool `json:"author_is_uploader"` -} - -type Chapter struct { - // The start time of the chapter in seconds - StartTime float64 `json:"start_time"` - - // The end time of the chapter in seconds - EndTime float64 `json:"end_time"` - - Title *string `json:"title"` -} diff --git a/ytdl/meta.go b/ytdl/meta.go deleted file mode 100644 index 81538d5..0000000 --- a/ytdl/meta.go +++ /dev/null @@ -1,36 +0,0 @@ -package ytdl - -import ( - "bytes" - "encoding/json" - "fmt" - "os/exec" - - "github.com/spf13/viper" -) - -func GetMetadata(url string) (Metadata, error) { - cmd := exec.Command( - viper.GetString("ytdlp_path"), - "-J", - "--cookies-from-browser", "firefox", - url, - ) - - var out bytes.Buffer - cmd.Stdout = &out - - if err := cmd.Run(); err != nil { - - fmt.Printf("%+v\n", err) - return Metadata{}, err - } - - var meta Metadata - if err := json.Unmarshal(out.Bytes(), &meta); err != nil { - fmt.Printf("%+v\n", err) - return Metadata{}, err - } - - return meta, nil -} diff --git a/ytdl/metadata/chapter.go b/ytdl/metadata/chapter.go new file mode 100644 index 0000000..749460f --- /dev/null +++ b/ytdl/metadata/chapter.go @@ -0,0 +1,11 @@ +package metadata + +type Chapter struct { + // The start time of the chapter in seconds + StartTime float64 `json:"start_time"` + + // The end time of the chapter in seconds + EndTime float64 `json:"end_time"` + + Title *string `json:"title"` +} diff --git a/ytdl/metadata/comment.go b/ytdl/metadata/comment.go new file mode 100644 index 0000000..ffeb3d4 --- /dev/null +++ b/ytdl/metadata/comment.go @@ -0,0 +1,43 @@ +package metadata + +type Comment struct { + // human-readable name of the comment author + Author *string `json:"author"` + + // user ID of the comment author + AuthorID *string `json:"author_id"` + + // The thumbnail of the comment author + AuthorThumbnail *string `json:"author_thumbnail"` + + // Comment ID + ID *string `json:"id"` + + // Comment as HTML + HTML *string `json:"html"` + + // Plain text of the comment + Text *string `json:"text"` + + // UNIX timestamp of comment + Timestamp *float64 `json:"timestamp"` + + // ID of the comment this one is replying to. + // Set to "root" to indicate that this is a + // comment to the original video. + Parent string `json:"parent"` + + // Number of positive ratings of the comment + LikeCount *int64 `json:"like_count"` + + // Number of negative ratings of the comment + DislikeCount *int64 `json:"dislike_count"` + + // Whether the comment is marked as + // favorite by the video uploader + IsFavorited bool `json:"is_favorited"` + + // Whether the comment is made by + // the video uploader + AuthorIsUploader bool `json:"author_is_uploader"` +} diff --git a/ytdl/metadata/format.go b/ytdl/metadata/format.go new file mode 100644 index 0000000..8a97f1d --- /dev/null +++ b/ytdl/metadata/format.go @@ -0,0 +1,160 @@ +package metadata + +type Format struct { + // The mandatory URL representing the media: + // for plain file media - HTTP URL of this file, + // for RTMP - RTMP URL, + // for HLS - URL of the M3U8 media playlist, + // for HDS - URL of the F4M manifest, + // for DASH + // - HTTP URL to plain file media (in case of + // unfragmented media) + // - URL of the MPD manifest or base URL + // representing the media if MPD manifest + // is parsed from a string (in case of + // fragmented media) + // for MSS - URL of the ISM manifest. + Url string `json:"url"` + + // Will be calculated from URL if missing + Ext string `json:"ext"` + + // A human-readable description of the format + // ("mp4 container with h264/opus"). + // Calculated from the format_id, width, height. + // and format_note fields if missing. + Format string `json:"format"` + + // A short description of the format + // ("mp4_h264_opus" or "19"). + // Technically optional, but strongly recommended. + FormatID string `json:"format_id"` + + // Additional info about the format + // ("3D" or "DASH video") + FormatNote string `json:"format_note"` + + // Width of the video, if known + Width int `json:"width"` + + // Height of the video, if known + Height int `json:"height"` + + // Aspect ratio of the video, if known + // Automatically calculated from width and height + AspectRatio float64 `json:"aspect_ratio"` + + // Textual description of width and height + // Automatically calculated from width and height + Resolution string `json:"resolution"` + + // The dynamic range of the video. One of: + // "SDR" (None), "HDR10", "HDR10+, "HDR12", "HLG, "DV" + DynamicRange string `json:"dynamic_range"` + + // Average bitrate of audio and video in KBit/s + Tbr float64 `json:"tbr"` + + // Average audio bitrate in KBit/s + Abr float64 `json:"abr"` + + // Average video bitrate in KBit/s + Vbr float64 `json:"vbr"` + + // Name of the audio codec in use + ACodec string `json:"acodec"` + + // Name of the video codec in use + VCodec string `json:"vcodec"` + + // Number of audio channels + AudioChannels int `json:"audio_channels"` + + // Frame rate + Fps float64 `json:"fps"` + + // Name of the container format + Container string `json:"container"` + + // The number of bytes, if known in advance + Filesize *int `json:"filesize"` + + // An estimate for the number of bytes + FilesizeApprox *int `json:"filesize_approx"` + + // The protocol that will be used for the actual + // download, lower-case. One of "http", "https" or + // one of the protocols defined in downloader.PROTOCOL_MAP + Protocol string `json:"protocol"` + + // Base URL for fragments. Each fragment's path + // value (if present) will be relative to + // this URL. + FragmentBaseUrl *string `json:"fragment_base_url"` + + // A list of fragments of a fragmented media. + // Each fragment entry must contain either an url + // or a path. If an url is present it should be + // considered by a client. Otherwise both path and + // fragment_base_url must be present. + Fragments []Fragment `json:"fragments"` + + // Is a live format that can be downloaded from the start. + IsFromStart bool `json:"is_from_start"` + + // Order number of this format. If this field is + // present and not None, the formats get sorted + // by this field, regardless of all other values. + // -1 for default (order by other properties), + // -2 or smaller for less than default. + // < -1000 to hide the format (if there is + // another one which is strictly better) + Preference *int `json:"preference"` + + // Language code, e.g. "de" or "en-US". + Language string `json:"language"` + + // Is this in the language mentioned in + // the URL? + // 10 if it's what the URL is about, + // -1 for default (don't know), + // -10 otherwise, other values reserved for now. + LanguagePreference int `json:"language_preference"` + + // Order number of the video quality of this + // format, irrespective of the file format. + // -1 for default (order by other properties), + // -2 or smaller for less than default. + Quality float64 `json:"quality"` + + // Order number for this video source + // (quality takes higher priority) + // -1 for default (order by other properties), + // -2 or smaller for less than default. + SourcePreference int `json:"source_preference"` + + // A dictionary of additional HTTP headers to add to the request. + HttpHeaders map[string]string `json:"http_header"` + + // If given and not 1, indicates that the + // video's pixels are not square. + // width : height ratio as float. + StretchedRatio *float64 `json:"stretched_ratio"` + + // The server does not support resuming the (HTTP or RTMP) download. + NoResume bool `json:"no_resume"` + + // The format has DRM and cannot be downloaded. + HasDrm bool `json:"has_drm"` + + // A query string to append to each + // fragment's URL, or to update each existing query string + // with. Only applied by the native HLS/DASH downloaders. + ExtraParamToSegmentUrl string `json:"extra_param_to_segment_url"` + + // A dictionary of HLS AES-128 decryption information + // used by the native HLS downloader to override the + // values in the media playlist when an '#EXT-X-KEY' tag + // is present in the playlist + HlsAes *HlsAes `json:"hls_aes"` +} diff --git a/ytdl/metadata/fragment.go b/ytdl/metadata/fragment.go new file mode 100644 index 0000000..31d4e50 --- /dev/null +++ b/ytdl/metadata/fragment.go @@ -0,0 +1,12 @@ +package metadata + +type Fragment struct { + // fragment's URL + Url string `json:"url"` + + // fragment's path relative to fragment_base_url + Path string `json:"path"` + + Duration *float64 `json:"duration"` + Filesize *int `json:"filesize"` +} diff --git a/ytdl/metadata/hls.go b/ytdl/metadata/hls.go new file mode 100644 index 0000000..27f786d --- /dev/null +++ b/ytdl/metadata/hls.go @@ -0,0 +1,13 @@ +package metadata + +type HlsAes struct { + // The URI from which the key will be downloaded + Uri string `json:"uri"` + + // The key (as hex) used to decrypt fragments. + // If `key` is given, any key URI will be ignored + Key string `json:"key"` + + // The IV (as hex) used to decrypt fragments + Iv string `json:"iv"` +} diff --git a/ytdl/metadata/metadata.go b/ytdl/metadata/metadata.go new file mode 100644 index 0000000..3e4bac0 --- /dev/null +++ b/ytdl/metadata/metadata.go @@ -0,0 +1,178 @@ +package metadata + +type Metadata struct { + Type string `json:"_type"` + + // A list of videos in the playlist + Entries []Metadata + + // Video identifier. + ID string `json:"id"` + + // Video title, unescaped. Set to an empty string if video has + // no title as opposed to "None" which signifies that the + // extractor failed to obtain a title + Title *string `json:"title"` + + // A list of dictionaries for each format available, ordered + // from worst to best quality. + Formats []Format `json:"formats"` + + // Final video URL. + Url string `json:"url"` + + // Video filename extension. + Ext string `json:"ext"` + + // The video format, defaults to ext (used for --get-format) + Format string `json:"format"` + + // True if a direct video file was given (must only be set by GenericIE) + Direct *bool `json:"direct"` + + // A secondary title of the video. + AltTitle *string `json:"alt_title"` + + // An alternative identifier for the video, not necessarily + // unique, but available before title. Typically, id is + // something like "4234987", title "Dancing naked mole rats", + // and display_id "dancing-naked-mole-rats" + DisplayID *string `json:"display_id"` + + Thumbnails []Thumbnail `json:"thumbnails"` + + // Full URL to a video thumbnail image. + Thumbnail *string `json:"thumbnail"` + + // Full video description. + Description *string `json:"description"` + + // Full name of the video uploader. + Uploader *string `json:"uploader"` + + // License name the video is licensed under. + License *string `json:"license"` + + // The creator of the video. + Creator *string `json:"creator"` + + // UNIX timestamp of the moment the video was uploaded + Timestamp *float64 `json:"timestamp"` + + // Video upload date in UTC (YYYYMMDD). + // If not explicitly set, calculated from timestamp + UploadDate *string `json:"upload_date"` + + // UNIX timestamp of the moment the video was released. + // If it is not clear whether to use timestamp or this, use the former + ReleaseTimestamp *float64 `json:"release_timestamp"` + + // The date (YYYYMMDD) when the video was released in UTC. + // If not explicitly set, calculated from release_timestamp + ReleaseDate *string `json:"release_date"` + + // UNIX timestamp of the moment the video was last modified. + ModifiedTimestamp *float64 `json:"modified_timestamp"` + + // The date (YYYYMMDD) when the video was last modified in UTC. + // If not explicitly set, calculated from modified_timestamp + ModifiedDate *string `json:"modified_date"` + + // Nickname or id of the video uploader. + UploaderId *string `json:"uploader_id"` + + // Full URL to a personal webpage of the video uploader. + UploaderUrl *string `json:"uploader_url"` + + // Full name of the channel the video is uploaded on. + // Note that channel fields may or may not repeat uploader + // fields. This depends on a particular extractor. + Channel *string `json:"channel"` + + // Id of the channel. + ChannelId *string `json:"channel_id"` + + // Full URL to a channel webpage. + ChannelUrl *string `json:"channel_url"` + + // Number of followers of the channel. + ChannelFollowerCount *int `json:"channel_follower_count"` + + // Physical location where the video was filmed. + Location *string `json:"location"` + + // The available subtitles as a dictionary in the format + // {tag: subformats}. "tag" is usually a language code, and + // "subformats" is a list sorted from lower to higher + // preference + Subtitles map[string][]Subtitle `json:"subtitles"` + + // Like 'subtitles'; contains automatically generated + // captions instead of normal subtitles + AutomaticCaptions map[string][]Subtitle `json:"automatic_captions"` + + // Length of the video in seconds, as an integer or float. + Duration *float64 `json:"duration"` + + // How many users have watched the video on the platform. + ViewCount *int64 `json:"view_count"` + + // How many users are currently watching the video on the platform. + ConcurrentViewCount *int64 `json:"concurrent_view_count"` + + // Number of positive ratings of the video + LikeCount *int64 `json:"like_count"` + + // Number of negative ratings of the video + DislikeCount *int64 `json:"dislike_count"` + + // Number of reposts of the video + RepostCount *int64 `json:"repost_count"` + + // Average rating give by users, the scale used depends on the webpage + AverageRating *float64 `json:"average_rating"` + + // Number of comments on the video + CommentCount *int64 `json:"comment_count"` + + // A list of comments + Comments []Comment `json:"comments"` + + // Age restriction for the video, as an integer (years) + AgeLimit *int `json:"age_limit"` + + // The URL to the video webpage, if given to yt-dlp it + // should allow to get the same result again. (It will be set + // by YoutubeDL if it's missing) + WebpageUrl *string `json:"webpage_url"` + + // A list of categories that the video falls in + Categories []string `json:"categories"` + + // A list of tags assigned to the video + Tags []string `json:"tags"` + + // A list of the video cast + Cast []string `json:"cast"` + + // Whether this video is a live stream that goes on instead of a fixed-length video. + IsLive *bool `json:"is_live"` + + // Whether this video was originally a live stream. + WasLive *bool `json:"was_live"` + + // None (=unknown), 'is_live', 'is_upcoming', 'was_live', 'not_live', + // or 'post_live' (was live, but VOD is not yet processed) + // If absent, automatically set from is_live, was_live + LiveStatus *string `json:"live_status"` + + // Time in seconds where the reproduction should start, as + // specified in the URL. + StartTime *float64 `json:"start_time"` + + // Time in seconds where the reproduction should end, as + // specified in the URL. + EndTime *float64 `json:"end_time"` + + Chapters []Chapter `json:"chapters"` +} diff --git a/ytdl/metadata/subtitle.go b/ytdl/metadata/subtitle.go new file mode 100644 index 0000000..5d86dc0 --- /dev/null +++ b/ytdl/metadata/subtitle.go @@ -0,0 +1,18 @@ +package metadata + +type Subtitle struct { + // Will be calculated from URL if missing + Ext string `json:"ext"` + + // The subtitles file contents + Data string `json:"data"` + + // A URL pointing to the subtitles file + Url string `json:"url"` + + // Name or description of the subtitles + Name string `json:"name"` + + // A dictionary of additional HTTP headers to add to the request. + HttpHeaders map[string]string `json:"http_headers"` +} diff --git a/ytdl/metadata/thumbnail.go b/ytdl/metadata/thumbnail.go new file mode 100644 index 0000000..5744f6d --- /dev/null +++ b/ytdl/metadata/thumbnail.go @@ -0,0 +1,18 @@ +package metadata + +type Thumbnail struct { + // Thumbnail format ID + ID *string `json:"id"` + + Url string `json:"url"` + + // Quality of the image + Preference *int `json:"preference"` + + Width *int `json:"width"` + Height *int `json:"height"` + Filesize *int `json:"filesize"` + + // HTTP headers for the request + HttpHeaders map[string]string `json:"http_headers"` +} diff --git a/ytdl/stream.go b/ytdl/stream.go deleted file mode 100644 index 6589bad..0000000 --- a/ytdl/stream.go +++ /dev/null @@ -1,26 +0,0 @@ -package ytdl - -import ( - "io" - "os/exec" - - "github.com/spf13/viper" -) - -func Stream(wr io.Writer, url string, format Format) error { - cmd := exec.Command( - viper.GetString("ytdlp_path"), - "-o", "-", - "-f", format.FormatID, - "--merge-output-format", "mkv", - "--cookies-from-browser", "firefox", - url, - ) - - cmd.Stdout = wr - if err := cmd.Run(); err != nil { - return err - } - - return nil -} diff --git a/ytdl/ytdl.go b/ytdl/ytdl.go new file mode 100644 index 0000000..077e7f5 --- /dev/null +++ b/ytdl/ytdl.go @@ -0,0 +1,87 @@ +package ytdl + +import ( + "io" + + "go.fifitido.net/ytdl-web/config" + "go.fifitido.net/ytdl-web/ytdl/cache" + "go.fifitido.net/ytdl-web/ytdl/metadata" + "golang.org/x/exp/slog" +) + +type Ytdl interface { + GetMetadata(url string) (*metadata.Metadata, error) + Download(w io.Writer, url, format string) error +} + +type ytdlImpl struct { + cfg *config.Config + logger *slog.Logger + cache cache.MetadataCache +} + +func NewYtdl(cfg *config.Config, logger *slog.Logger, cache cache.MetadataCache) Ytdl { + return &ytdlImpl{ + cfg: cfg, + logger: logger.With(slog.String("module", "ytdl")), + cache: cache, + } +} + +func (y *ytdlImpl) baseOptions(url string) []Option { + options := []Option{} + + metadata, err := y.cache.Get(url) + if err == nil { + options = append(options, WithLoadJson(metadata)) + } + + if y.cfg.Cookies.Enabled { + if y.cfg.Cookies.FromBrowser != nil { + options = append(options, WithBrowserCookies( + y.cfg.Cookies.FromBrowser.Browser, + y.cfg.Cookies.FromBrowser.Keyring, + y.cfg.Cookies.FromBrowser.Profile, + y.cfg.Cookies.FromBrowser.Container, + )) + } else { + options = append(options, WithCookieFile(y.cfg.Cookies.FilePath)) + } + } + + return options +} + +// GetMetadata implements Ytdl +func (y *ytdlImpl) GetMetadata(url string) (*metadata.Metadata, error) { + meta := &metadata.Metadata{} + options := append( + y.baseOptions(url), + WithDumpJson(meta), + ) + + if err := Exec(url, options...); err != nil { + return nil, err + } + + if err := y.cache.Set(url, meta, y.cfg.Ytdlp.Cache.TTL); err != nil { + y.logger.Warn("failed to cache metadata", slog.String("url", url), slog.String("error", err.Error())) + } + + return meta, nil +} + +// Download implements Ytdl +func (y *ytdlImpl) Download(w io.Writer, url, format string) error { + options := append( + y.baseOptions(url), + WithFormat(format), + WithStreamOutput(w), + ) + + if err := Exec(url, options...); err != nil { + return err + } + + return nil +}