Use custom ytdl wrapper

This commit is contained in:
Evan Fiordeliso 2023-04-14 16:07:57 -04:00
parent 177d8d7e55
commit 05f23c53ec
9 changed files with 559 additions and 8 deletions

2
go.mod
View File

@ -5,7 +5,9 @@ go 1.20
require (
github.com/gofiber/fiber/v2 v2.43.0
github.com/gofiber/template v1.8.0
github.com/samber/lo v1.38.1
github.com/spf13/cobra v1.7.0
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
)
require (

4
go.sum
View File

@ -331,6 +331,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM=
github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94 h1:rmMl4fXJhKMNWl+K+r/fq4FbbKI+Ia2m9hYBLm2h4G4=
github.com/savsgio/dictpool v0.0.0-20221023140959-7bf2e61cea94/go.mod h1:90zrgN3D/WJsDd1iXHT96alCoN2KJo6/4x1DZC3wZs8=
github.com/savsgio/gotils v0.0.0-20220530130905-52f3993e8d6d/go.mod h1:Gy+0tqhJvgGlqnTF8CVGP0AaGRjwBtXs/a5PA0Y3+A4=
@ -413,6 +415,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

View File

@ -1,7 +1,13 @@
package web
import (
"net/url"
"sort"
"github.com/gofiber/fiber/v2"
"github.com/samber/lo"
"go.fifitido.net/ytdl-web/ytdl"
"golang.org/x/exp/slog"
)
func Serve() error {
@ -13,10 +19,29 @@ func Serve() error {
})
app.Get("/download", func(c *fiber.Ctx) error {
url := c.Get("url")
urlBytes, err := url.QueryUnescape(c.Query("url"))
if err != nil {
slog.Error("Failed to decode url param", slog.String("error", err.Error()))
return fiber.ErrBadRequest
}
url := string(urlBytes)
meta, err := ytdl.GetMetadata(url)
if err != nil {
slog.Error("Failed to get metadata", slog.String("error", err.Error()))
return fiber.ErrInternalServerError
}
formats := lo.Filter(meta.Formats, func(item ytdl.Format, _ int) bool {
return item.ACodec != "none" && item.VCodec != "none"
})
sort.Slice(formats, func(i, j int) bool {
return formats[i].Width > formats[j].Width
})
return c.Render("views/download", fiber.Map{
"Url": url,
"Url": url, "Meta": meta, "Formats": formats,
}, "views/layouts/main")
})

View File

@ -2,8 +2,10 @@ package web
import (
"embed"
"encoding/json"
"html/template"
"net/http"
"net/url"
"github.com/gofiber/template/html"
)
@ -14,10 +16,23 @@ var viewsfs embed.FS
func ViewsEngine() *html.Engine {
engine := html.NewFileSystem(http.FS(viewsfs), ".html")
engine.AddFunc(
// add unescape function
"unescape", func(s string) template.HTML {
"unsafe", func(s string) template.HTML {
return template.HTML(s)
},
)
engine.AddFunc(
"queryEscape", func(s string) string {
return url.QueryEscape(s)
},
)
engine.AddFunc(
"jsonMarshal", func(s any) (string, error) {
j, err := json.MarshalIndent(s, "", " ")
if err != nil {
return "", err
}
return string(j), nil
},
)
return engine
}

View File

@ -1,2 +1,37 @@
<h1>Download</h1>
<p>{{.Url}}</p>
<div class="d-flex flex-column align-items-center">
<h1>Download Video</h1>
<h2 class="fs-4 text-muted">{{.Meta.Title}}</h2>
<p style="font-size: 0.85rem">{{.Url}}</p>
<img
src="{{.Meta.Thumbnail}}"
alt="{{.Meta.Title}}"
style="max-height: 25rem; max-width: 100%"
/>
<a
href="/"
class="btn btn-secondary btn-sm mt-3"
style="width: 30rem; max-width: 100%"
>Download Another Video</a
>
</div>
{{$id := .Meta.ID}} {{$url := .Url}} {{range .Formats}}
<div class="d-flex gap-3 mt-5 align-items-center">
<div style="width: 10rem">{{.Format}}</div>
<div class="flex-grow-1 d-flex gap-3">
<a
class="btn btn-primary flex-grow-1"
download="{{$id}}-{{.Resolution}}.{{.Ext}}"
href="{{.Url}}"
>
Download (direct)
</a>
<a
class="btn btn-primary flex-grow-1"
download="{{$id}}-{{.Resolution}}.{{.Ext}}"
href="/download/proxy?url={{queryEscape $url}}&format={{.FormatID}}"
>
Download (proxied)
</a>
</div>
</div>
{{end}}

View File

@ -7,7 +7,7 @@
<div class="mb-3">
<label for="url" class="form-label visually-hidden">Url</label>
<div class="input-group">
<input type="url" name="url" id="url" class="form-control" />
<input type="url" name="url" id="url" class="form-control" required />
<button
id="paste-button"
class="btn btn-outline-secondary"

View File

@ -46,7 +46,7 @@
</head>
<body data-bs-theme="dark">
{{template "views/partials/navbar" .}}
<main class="container mt-5">{{embed}}</main>
<main class="container my-5">{{embed}}</main>
<script>
const pasteButton = document.getElementById("paste-button");
const urlField = document.getElementById("url");

441
ytdl/data.go Normal file
View File

@ -0,0 +1,441 @@
package ytdl
type Metdata struct {
// Video identifier.
ID string `json:"id"`
// Video title, unescaped. Set to an empty string if video has
// no title as opposed to "None" which signifies that the
// extractor failed to obtain a title
Title *string `json:"title"`
// A list of dictionaries for each format available, ordered
// from worst to best quality.
Formats []Format `json:"formats"`
// Final video URL.
Url string `json:"url"`
// Video filename extension.
Ext string `json:"ext"`
// The video format, defaults to ext (used for --get-format)
Format string `json:"format"`
// True if a direct video file was given (must only be set by GenericIE)
Direct *bool `json:"direct"`
// A secondary title of the video.
AltTitle *string `json:"alt_title"`
// An alternative identifier for the video, not necessarily
// unique, but available before title. Typically, id is
// something like "4234987", title "Dancing naked mole rats",
// and display_id "dancing-naked-mole-rats"
DisplayID *string `json:"display_id"`
Thumbnails []Thumbnail `json:"thumbnails"`
// Full URL to a video thumbnail image.
Thumbnail *string `json:"thumbnail"`
// Full video description.
Description *string `json:"description"`
// Full name of the video uploader.
Uploader *string `json:"uploader"`
// License name the video is licensed under.
License *string `json:"license"`
// The creator of the video.
Creator *string `json:"creator"`
// UNIX timestamp of the moment the video was uploaded
Timestamp *int64 `json:"timestamp"`
// Video upload date in UTC (YYYYMMDD).
// If not explicitly set, calculated from timestamp
UploadDate *string `json:"upload_date"`
// UNIX timestamp of the moment the video was released.
// If it is not clear whether to use timestamp or this, use the former
ReleaseTimestamp *int64 `json:"release_timestamp"`
// The date (YYYYMMDD) when the video was released in UTC.
// If not explicitly set, calculated from release_timestamp
ReleaseDate *string `json:"release_date"`
// UNIX timestamp of the moment the video was last modified.
ModifiedTimestamp *int64 `json:"modified_timestamp"`
// The date (YYYYMMDD) when the video was last modified in UTC.
// If not explicitly set, calculated from modified_timestamp
ModifiedDate *string `json:"modified_date"`
// Nickname or id of the video uploader.
UploaderId *string `json:"uploader_id"`
// Full URL to a personal webpage of the video uploader.
UploaderUrl *string `json:"uploader_url"`
// Full name of the channel the video is uploaded on.
// Note that channel fields may or may not repeat uploader
// fields. This depends on a particular extractor.
Channel *string `json:"channel"`
// Id of the channel.
ChannelId *string `json:"channel_id"`
// Full URL to a channel webpage.
ChannelUrl *string `json:"channel_url"`
// Number of followers of the channel.
ChannelFollowerCount *int `json:"channel_follower_count"`
// Physical location where the video was filmed.
Location *string `json:"location"`
// The available subtitles as a dictionary in the format
// {tag: subformats}. "tag" is usually a language code, and
// "subformats" is a list sorted from lower to higher
// preference
Subtitles map[string][]Subtitle `json:"subtitles"`
// Like 'subtitles'; contains automatically generated
// captions instead of normal subtitles
AutomaticCaptions map[string][]Subtitle `json:"automatic_captions"`
// Length of the video in seconds, as an integer or float.
Duration *float64 `json:"duration"`
// How many users have watched the video on the platform.
ViewCount *int64 `json:"view_count"`
// How many users are currently watching the video on the platform.
ConcurrentViewCount *int64 `json:"concurrent_view_count"`
// Number of positive ratings of the video
LikeCount *int64 `json:"like_count"`
// Number of negative ratings of the video
DislikeCount *int64 `json:"dislike_count"`
// Number of reposts of the video
RepostCount *int64 `json:"repost_count"`
// Average rating give by users, the scale used depends on the webpage
AverageRating *float64 `json:"average_rating"`
// Number of comments on the video
CommentCount *int64 `json:"comment_count"`
// A list of comments
Comments []Comment `json:"comments"`
// Age restriction for the video, as an integer (years)
AgeLimit *int `json:"age_limit"`
// The URL to the video webpage, if given to yt-dlp it
// should allow to get the same result again. (It will be set
// by YoutubeDL if it's missing)
WebpageUrl *string `json:"webpage_url"`
// A list of categories that the video falls in
Categories []string `json:"categories"`
// A list of tags assigned to the video
Tags []string `json:"tags"`
// A list of the video cast
Cast []string `json:"cast"`
// Whether this video is a live stream that goes on instead of a fixed-length video.
IsLive *bool `json:"is_live"`
// Whether this video was originally a live stream.
WasLive *bool `json:"was_live"`
// None (=unknown), 'is_live', 'is_upcoming', 'was_live', 'not_live',
// or 'post_live' (was live, but VOD is not yet processed)
// If absent, automatically set from is_live, was_live
LiveStatus *string `json:"live_status"`
// Time in seconds where the reproduction should start, as
// specified in the URL.
StartTime *int64 `json:"start_time"`
// Time in seconds where the reproduction should end, as
// specified in the URL.
EndTime *int64 `json:"end_time"`
Chapters []Chapter `json:"chapters"`
}
type Format struct {
// The mandatory URL representing the media:
// for plain file media - HTTP URL of this file,
// for RTMP - RTMP URL,
// for HLS - URL of the M3U8 media playlist,
// for HDS - URL of the F4M manifest,
// for DASH
// - HTTP URL to plain file media (in case of
// unfragmented media)
// - URL of the MPD manifest or base URL
// representing the media if MPD manifest
// is parsed from a string (in case of
// fragmented media)
// for MSS - URL of the ISM manifest.
Url string `json:"url"`
// Will be calculated from URL if missing
Ext string `json:"ext"`
// A human-readable description of the format
// ("mp4 container with h264/opus").
// Calculated from the format_id, width, height.
// and format_note fields if missing.
Format string `json:"format"`
// A short description of the format
// ("mp4_h264_opus" or "19").
// Technically optional, but strongly recommended.
FormatID string `json:"format_id"`
// Additional info about the format
// ("3D" or "DASH video")
FormatNote string `json:"format_note"`
// Width of the video, if known
Width int `json:"width"`
// Height of the video, if known
Height int `json:"height"`
// Aspect ratio of the video, if known
// Automatically calculated from width and height
AspectRatio float64 `json:"aspect_ratio"`
// Textual description of width and height
// Automatically calculated from width and height
Resolution string `json:"resolution"`
// The dynamic range of the video. One of:
// "SDR" (None), "HDR10", "HDR10+, "HDR12", "HLG, "DV"
DynamicRange string `json:"dynamic_range"`
// Average bitrate of audio and video in KBit/s
Tbr float64 `json:"tbr"`
// Average audio bitrate in KBit/s
Abr float64 `json:"abr"`
// Average video bitrate in KBit/s
Vbr float64 `json:"vbr"`
// Name of the audio codec in use
ACodec string `json:"acodec"`
// Name of the video codec in use
VCodec string `json:"vcodec"`
// Number of audio channels
AudioChannels int `json:"audio_channels"`
// Frame rate
Fps float64 `json:"fps"`
// Name of the container format
Container string `json:"container"`
// The number of bytes, if known in advance
Filesize *int `json:"filesize"`
// An estimate for the number of bytes
FilesizeApprox int `json:"filesize_approx"`
// The protocol that will be used for the actual
// download, lower-case. One of "http", "https" or
// one of the protocols defined in downloader.PROTOCOL_MAP
Protocol string `json:"protocol"`
// Base URL for fragments. Each fragment's path
// value (if present) will be relative to
// this URL.
FragmentBaseUrl *string `json:"fragment_base_url"`
// A list of fragments of a fragmented media.
// Each fragment entry must contain either an url
// or a path. If an url is present it should be
// considered by a client. Otherwise both path and
// fragment_base_url must be present.
Fragments []Fragment `json:"fragments"`
// Is a live format that can be downloaded from the start.
IsFromStart bool `json:"is_from_start"`
// Order number of this format. If this field is
// present and not None, the formats get sorted
// by this field, regardless of all other values.
// -1 for default (order by other properties),
// -2 or smaller for less than default.
// < -1000 to hide the format (if there is
// another one which is strictly better)
Preference *int `json:"preference"`
// Language code, e.g. "de" or "en-US".
Language string `json:"language"`
// Is this in the language mentioned in
// the URL?
// 10 if it's what the URL is about,
// -1 for default (don't know),
// -10 otherwise, other values reserved for now.
LanguagePreference int `json:"language_preference"`
// Order number of the video quality of this
// format, irrespective of the file format.
// -1 for default (order by other properties),
// -2 or smaller for less than default.
Quality float64 `json:"quality"`
// Order number for this video source
// (quality takes higher priority)
// -1 for default (order by other properties),
// -2 or smaller for less than default.
SourcePreference int `json:"source_preference"`
// A dictionary of additional HTTP headers to add to the request.
HttpHeaders map[string]string `json:"http_header"`
// If given and not 1, indicates that the
// video's pixels are not square.
// width : height ratio as float.
StretchedRatio *float64 `json:"stretched_ratio"`
// The server does not support resuming the (HTTP or RTMP) download.
NoResume bool `json:"no_resume"`
// The format has DRM and cannot be downloaded.
HasDrm bool `json:"has_drm"`
// A query string to append to each
// fragment's URL, or to update each existing query string
// with. Only applied by the native HLS/DASH downloaders.
ExtraParamToSegmentUrl string `json:"extra_param_to_segment_url"`
// A dictionary of HLS AES-128 decryption information
// used by the native HLS downloader to override the
// values in the media playlist when an '#EXT-X-KEY' tag
// is present in the playlist
HlsAes *HlsAes `json:"hls_aes"`
}
type Fragment struct {
// fragment's URL
Url string `json:"url"`
// fragment's path relative to fragment_base_url
Path string `json:"path"`
Duration *float64 `json:"duration"`
Filesize *int `json:"filesize"`
}
type HlsAes struct {
// The URI from which the key will be downloaded
Uri string `json:"uri"`
// The key (as hex) used to decrypt fragments.
// If `key` is given, any key URI will be ignored
Key string `json:"key"`
// The IV (as hex) used to decrypt fragments
Iv string `json:"iv"`
}
type Thumbnail struct {
// Thumbnail format ID
ID *string `json:"id"`
Url string `json:"url"`
// Quality of the image
Preference *int `json:"preference"`
Width *int `json:"width"`
Height *int `json:"height"`
Filesize *int `json:"filesize"`
// HTTP headers for the request
HttpHeaders map[string]string `json:"http_headers"`
}
type Subtitle struct {
// Will be calculated from URL if missing
Ext string `json:"ext"`
// The subtitles file contents
Data string `json:"data"`
// A URL pointing to the subtitles file
Url string `json:"url"`
// Name or description of the subtitles
Name string `json:"name"`
// A dictionary of additional HTTP headers to add to the request.
HttpHeaders map[string]string `json:"http_headers"`
}
type Comment struct {
// human-readable name of the comment author
Author *string `json:"author"`
// user ID of the comment author
AuthorID *string `json:"author_id"`
// The thumbnail of the comment author
AuthorThumbnail *string `json:"author_thumbnail"`
// Comment ID
ID *string `json:"id"`
// Comment as HTML
HTML *string `json:"html"`
// Plain text of the comment
Text *string `json:"text"`
// UNIX timestamp of comment
Timestamp *int64 `json:"timestamp"`
// ID of the comment this one is replying to.
// Set to "root" to indicate that this is a
// comment to the original video.
Parent string `json:"parent"`
// Number of positive ratings of the comment
LikeCount *int64 `json:"like_count"`
// Number of negative ratings of the comment
DislikeCount *int64 `json:"dislike_count"`
// Whether the comment is marked as
// favorite by the video uploader
IsFavorited bool `json:"is_favorited"`
// Whether the comment is made by
// the video uploader
AuthorIsUploader bool `json:"author_is_uploader"`
}
type Chapter struct {
// The start time of the chapter in seconds
StartTime int64 `json:"start_time"`
// The end time of the chapter in seconds
EndTime int64 `json:"end_time"`
Title *string `json:"title"`
}

29
ytdl/meta.go Normal file
View File

@ -0,0 +1,29 @@
package ytdl
import (
"bytes"
"encoding/json"
"os/exec"
)
func GetMetadata(url string) (Metdata, error) {
cmd := exec.Command(
"yt-dlp",
"-J",
url,
)
var out bytes.Buffer
cmd.Stdout = &out
if err := cmd.Run(); err != nil {
return Metdata{}, nil
}
var meta Metdata
if err := json.Unmarshal(out.Bytes(), &meta); err != nil {
return Metdata{}, err
}
return meta, nil
}