feat: initial implementation of uptodownbot

Telegram bot for downloading media from any site using yt-dlp.
Supports audio/video/both, quality selection, language picker,
container format selection, playlist pagination, progress updates,
per-user quotas, user preferences, download history, cancel at any step,
polling and webhook modes, proxy support, and SQLite persistence.
This commit is contained in:
2026-06-25 14:22:48 +03:30
commit 05fcdf7458
26 changed files with 3302 additions and 0 deletions

78
internal/dl/progress.go Normal file
View File

@@ -0,0 +1,78 @@
package dl
import (
"bufio"
"io"
"regexp"
"strconv"
"strings"
"uptodownBot/internal/domain"
)
var progressRE = regexp.MustCompile(`\[\w+\]\s+([\d.]+)%\s+of\s+~?([\d.]+)(\w+)`)
type progressUpdate struct {
pct float64
speed string
eta string
}
// parseProgressLine parses a single yt-dlp stderr progress line.
// Returns nil if the line doesn't contain progress info.
func parseProgressLine(line string) *progressUpdate {
line = strings.TrimSpace(line)
if line == "" || !strings.HasPrefix(line, "[download]") {
return nil
}
// Try to extract the simple percentage from the default format:
// [download] 45.2% of 123.45MiB at 2.34MiB/s ETA 00:45
m := progressRE.FindStringSubmatch(line)
if m == nil {
return nil
}
pct, err := strconv.ParseFloat(m[1], 64)
if err != nil {
return nil
}
u := &progressUpdate{pct: pct}
// Extract speed and ETA from remaining parts
rest := line
if idx := strings.Index(rest, "%"); idx >= 0 {
rest = rest[idx+1:]
}
// Look for "at <speed>"
if atIdx := strings.Index(rest, " at "); atIdx >= 0 {
afterAt := rest[atIdx+4:]
if spaceIdx := strings.Index(afterAt, " "); spaceIdx >= 0 {
u.speed = strings.TrimSpace(afterAt[:spaceIdx])
rest = afterAt[spaceIdx:]
}
}
// Look for "ETA <duration>"
if etaIdx := strings.Index(rest, "ETA "); etaIdx >= 0 {
etaStr := strings.TrimSpace(rest[etaIdx+4:])
u.eta = etaStr
}
return u
}
// readProgress reads yt-dlp stderr and sends progress updates to the channel.
func readProgress(stderr io.ReadCloser, updates chan<- *domain.DownloadJob, job *domain.DownloadJob) {
defer close(updates)
scanner := bufio.NewScanner(stderr)
for scanner.Scan() {
line := scanner.Text()
parsed := parseProgressLine(line)
if parsed == nil {
continue
}
job.Progress = parsed.pct
job.Speed = parsed.speed
job.ETA = parsed.eta
updates <- job
}
}

View File

@@ -0,0 +1,72 @@
package dl
import (
"math"
"testing"
)
func TestParseProgressLine(t *testing.T) {
tests := []struct {
line string
pct float64
speed string
eta string
hasRes bool
}{
{
line: "[download] 45.2% of 123.45MiB at 2.34MiB/s ETA 00:45",
pct: 45.2,
speed: "2.34MiB/s",
eta: "00:45",
hasRes: true,
},
{
line: "[download] 10.0% of ~500.00MiB at 1.50MiB/s ETA 05:30",
pct: 10.0,
speed: "1.50MiB/s",
eta: "05:30",
hasRes: true,
},
{
line: "[download] 100.0% of 50.00MiB at 5.00MiB/s ETA 00:00",
pct: 100.0,
speed: "5.00MiB/s",
eta: "00:00",
hasRes: true,
},
{
line: "[youtube] Extracting URL: https://example.com",
hasRes: false,
},
{
line: "",
hasRes: false,
},
{
line: "some random log line",
hasRes: false,
},
}
for _, tt := range tests {
got := parseProgressLine(tt.line)
if !tt.hasRes {
if got != nil {
t.Errorf("parseProgressLine(%q) = %+v, want nil", tt.line, got)
}
continue
}
if got == nil {
t.Errorf("parseProgressLine(%q) = nil, want result", tt.line)
continue
}
if math.Abs(got.pct-tt.pct) > 0.01 {
t.Errorf("parseProgressLine(%q).pct = %f, want %f", tt.line, got.pct, tt.pct)
}
if got.speed != tt.speed {
t.Errorf("parseProgressLine(%q).speed = %q, want %q", tt.line, got.speed, tt.speed)
}
if got.eta != tt.eta {
t.Errorf("parseProgressLine(%q).eta = %q, want %q", tt.line, got.eta, tt.eta)
}
}
}

411
internal/dl/ytdlp.go Normal file
View File

@@ -0,0 +1,411 @@
package dl
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"log/slog"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
"time"
"uptodownBot/internal/domain"
)
// Client wraps calls to the yt-dlp executable.
type Client struct {
binaryPath string
cookiesFile string
}
// NewClient creates a new yt-dlp client. It checks for the binary.
func NewClient(cookiesFile string) (*Client, error) {
path, err := exec.LookPath("yt-dlp")
if err != nil {
return nil, fmt.Errorf("yt-dlp not found in PATH: %w", err)
}
slog.Info("yt-dlp found", "path", path)
return &Client{binaryPath: path, cookiesFile: cookiesFile}, nil
}
type ytdlpFormat struct {
FormatID string `json:"format_id"`
Ext string `json:"ext"`
Width int `json:"width"`
Height int `json:"height"`
Filesize int64 `json:"filesize"`
FilesizeApprox int64 `json:"filesize_approx"`
Codec string `json:"vcodec"`
ACodec string `json:"acodec"`
TBR float64 `json:"tbr"`
ABR float64 `json:"abr"`
Language string `json:"language"`
Resolution string `json:"resolution"`
}
type ytdlpInfo struct {
Title string `json:"title"`
Duration float64 `json:"duration"`
IsLive bool `json:"is_live"`
Formats []ytdlpFormat `json:"formats"`
Playlist string `json:"playlist"`
PlaylistID string `json:"playlist_id"`
PlaylistCount int `json:"playlist_count"`
Entries []json.RawMessage `json:"entries"`
RequestedFormats []ytdlpFormat `json:"requested_formats"`
}
type ytdlpPlaylistEntry struct {
ID string `json:"id"`
Title string `json:"title"`
URL string `json:"url"`
Duration float64 `json:"duration"`
}
// GetMediaInfo fetches format information for a given URL.
func (c *Client) GetMediaInfo(url string) (*domain.MediaInfo, error) {
args := []string{
"-J", "--no-download",
"--no-warnings",
url,
}
if c.cookiesFile != "" {
args = append(args, "--cookies", c.cookiesFile)
}
slog.Info("fetching media info", "url", url)
out, err := c.run(args)
if err != nil {
return nil, fmt.Errorf("yt-dlp info failed: %w", err)
}
var info ytdlpInfo
if err := json.Unmarshal(out, &info); err != nil {
return nil, fmt.Errorf("parse yt-dlp output: %w", err)
}
if info.IsLive {
return nil, fmt.Errorf("cannot download live streams")
}
mi := &domain.MediaInfo{
URL: url,
Title: info.Title,
Duration: info.Duration,
IsPlaylist: info.Playlist != "",
}
if mi.IsPlaylist {
mi.PlaylistCount = info.PlaylistCount
mi.PlaylistEntries = c.parsePlaylistEntries(info.Entries)
return mi, nil
}
langSet := make(map[string]bool)
for _, f := range info.Formats {
ff := c.convertFormat(f)
mi.Formats = append(mi.Formats, ff)
if ff.Language != "" && !langSet[ff.Language] {
langSet[ff.Language] = true
mi.Languages = append(mi.Languages, ff.Language)
}
if ff.Filesize > 0 {
mi.EstimatedSize += ff.Filesize
}
}
return mi, nil
}
// GetPlaylistInfo fetches a flat playlist listing (fast, no format detail).
func (c *Client) GetPlaylistInfo(url string) (*domain.MediaInfo, error) {
args := []string{
"-J", "--flat-playlist", "--no-download",
"--no-warnings",
url,
}
if c.cookiesFile != "" {
args = append(args, "--cookies", c.cookiesFile)
}
slog.Info("fetching playlist info", "url", url)
out, err := c.run(args)
if err != nil {
return nil, fmt.Errorf("yt-dlp playlist info failed: %w", err)
}
var info ytdlpInfo
if err := json.Unmarshal(out, &info); err != nil {
return nil, fmt.Errorf("parse yt-dlp playlist output: %w", err)
}
mi := &domain.MediaInfo{
URL: url,
Title: info.Title,
IsPlaylist: true,
PlaylistCount: info.PlaylistCount,
PlaylistEntries: c.parsePlaylistEntries(info.Entries),
}
return mi, nil
}
func (c *Client) parsePlaylistEntries(entries []json.RawMessage) []domain.PlaylistEntry {
var result []domain.PlaylistEntry
for _, raw := range entries {
var entry ytdlpPlaylistEntry
if err := json.Unmarshal(raw, &entry); err != nil {
slog.Warn("failed to parse playlist entry", "error", err)
continue
}
result = append(result, domain.PlaylistEntry{
ID: entry.ID,
Title: entry.Title,
URL: entry.URL,
Duration: entry.Duration,
})
}
return result
}
func (c *Client) convertFormat(f ytdlpFormat) domain.Format {
ff := domain.Format{
ID: f.FormatID,
Extension: f.Ext,
Width: f.Width,
Height: f.Height,
Resolution: f.Resolution,
Codec: f.Codec,
Language: f.Language,
}
if f.Filesize > 0 {
ff.Filesize = f.Filesize
} else {
ff.Filesize = f.FilesizeApprox
}
if f.TBR > 0 {
ff.Bitrate = fmt.Sprintf("%.0fk", f.TBR)
} else if f.ABR > 0 {
ff.Bitrate = fmt.Sprintf("%.0fk", f.ABR)
}
ff.HasVideo = f.Codec != "none" && f.Codec != ""
ff.HasAudio = f.ACodec != "none" && f.ACodec != ""
ff.IsAudioOnly = !ff.HasVideo && ff.HasAudio
ff.IsVideoOnly = ff.HasVideo && !ff.HasAudio
return ff
}
// StartDownload begins a download and sends progress updates to the channel.
// Returns the job and an updates channel. Caller must consume from updates until closed.
func (c *Client) StartDownload(job *domain.DownloadJob, downloadDir string) (<-chan *domain.DownloadJob, error) {
if err := os.MkdirAll(downloadDir, 0755); err != nil {
return nil, fmt.Errorf("create download dir: %w", err)
}
formatID := job.SelectedFormat.ID
var args []string
// Build format string for merge
switch job.MediaType {
case domain.MediaTypeAudio:
args = append(args, "-f", formatID, "-x")
if job.Container != "" {
args = append(args, "--audio-format", job.Container)
}
case domain.MediaTypeVideo:
args = append(args, "-f", formatID)
case domain.MediaTypeVideoAudio:
// Download best video + best audio that match
args = append(args, "-f", formatID+"+bestaudio/best")
if job.Container != "" {
args = append(args, "--merge-output-format", job.Container)
}
}
// Output template
outputPath := filepath.Join(downloadDir, "%(id)s.%(ext)s")
args = append(args, "--output", outputPath)
// Progress reporting
args = append(args, "--newline", "--no-progress")
// Language selection if specified
if job.Language != "" {
args = append(args, "--sub-langs", job.Language)
args = append(args, "--audio-multi-streams")
}
// Cookies
if c.cookiesFile != "" {
args = append(args, "--cookies", c.cookiesFile)
}
args = append(args, job.URL)
slog.Info("starting download", "url", job.URL, "format", formatID)
cmd := exec.Command(c.binaryPath, args...)
stderr, err := cmd.StderrPipe()
if err != nil {
return nil, fmt.Errorf("create stderr pipe: %w", err)
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("start yt-dlp: %w", err)
}
job.Cmd = cmd
job.CancelCh = make(chan struct{})
updates := make(chan *domain.DownloadJob, 100)
go func() {
defer func() {
if job.Status != domain.StatusCancelled {
job.Status = domain.StatusCompleted
updates <- job
}
close(updates)
}()
progressDone := make(chan struct{})
go func() {
readProgress(stderr, updates, job)
close(progressDone)
}()
// Wait for either completion or cancellation
done := make(chan error, 1)
go func() {
done <- cmd.Wait()
}()
select {
case <-job.CancelCh:
slog.Info("cancelling download", "url", job.URL)
cmd.Process.Signal(syscall.SIGTERM)
select {
case <-done:
case <-time.After(3 * time.Second):
cmd.Process.Kill()
}
<-progressDone
job.Status = domain.StatusCancelled
case err := <-done:
<-progressDone
if err != nil {
job.Status = domain.StatusFailed
slog.Error("download failed", "url", job.URL, "error", err)
} else {
job.Status = domain.StatusCompleted
}
}
}()
return updates, nil
}
// CancelDownload sends a cancellation signal to a running download.
func CancelDownload(job *domain.DownloadJob) {
if job == nil {
return
}
select {
case job.CancelCh <- struct{}{}:
default:
// Already cancelled or not started
}
if job.Cmd != nil && job.Cmd.Process != nil {
job.Cmd.Process.Signal(syscall.SIGTERM)
}
}
func (c *Client) run(args []string) ([]byte, error) {
cmd := exec.Command(c.binaryPath, args...)
var stderr bytes.Buffer
cmd.Stderr = &stderr
out, err := cmd.Output()
if err != nil {
stderrStr := strings.TrimSpace(stderr.String())
slog.Warn("yt-dlp run failed", "stderr", stderrStr, "error", err)
// Extract user-friendly error message
if stderrStr != "" {
return nil, errors.New(stderrStr)
}
return nil, err
}
return out, nil
}
// FilterFormatsByType filters formats by the requested media type.
func FilterFormatsByType(formats []domain.Format, mt domain.MediaType) []domain.Format {
var filtered []domain.Format
for _, f := range formats {
switch mt {
case domain.MediaTypeAudio:
if f.IsAudioOnly {
filtered = append(filtered, f)
}
case domain.MediaTypeVideo:
if f.IsVideoOnly {
filtered = append(filtered, f)
}
case domain.MediaTypeVideoAudio:
if f.HasVideo && f.HasAudio {
filtered = append(filtered, f)
}
}
}
return filtered
}
// DeduplicateResolutions reduces formats to unique resolution labels.
func DeduplicateResolutions(formats []domain.Format) []domain.Format {
seen := make(map[string]bool)
var result []domain.Format
for _, f := range formats {
label := formatLabel(f)
if !seen[label] {
seen[label] = true
result = append(result, f)
}
}
return result
}
// formatLabel returns a human-readable label for a format.
func formatLabel(f domain.Format) string {
if f.IsAudioOnly {
if f.Bitrate != "" {
return f.Bitrate
}
return "audio"
}
if f.Resolution != "" {
return f.Resolution
}
if f.Height > 0 {
return fmt.Sprintf("%dp", f.Height)
}
return f.ID
}
// ContainerOptions returns the container format options for a given media type.
func ContainerOptions(mt domain.MediaType) []string {
switch mt {
case domain.MediaTypeAudio:
return []string{"mp3", "m4a", "opus"}
case domain.MediaTypeVideo:
return []string{"mp4", "mkv", "webm"}
case domain.MediaTypeVideoAudio:
return []string{"mp4", "mkv"}
default:
return []string{"mp4"}
}
}

103
internal/dl/ytdlp_test.go Normal file
View File

@@ -0,0 +1,103 @@
package dl
import (
"testing"
"uptodownBot/internal/domain"
)
func TestFilterFormatsByType(t *testing.T) {
formats := []domain.Format{
{ID: "1", HasVideo: true, HasAudio: false, IsVideoOnly: true, IsAudioOnly: false},
{ID: "2", HasVideo: false, HasAudio: true, IsVideoOnly: false, IsAudioOnly: true},
{ID: "3", HasVideo: true, HasAudio: true, IsVideoOnly: false, IsAudioOnly: false},
{ID: "4", HasVideo: true, HasAudio: false, IsVideoOnly: true, IsAudioOnly: false},
{ID: "5", HasVideo: false, HasAudio: true, IsVideoOnly: false, IsAudioOnly: true},
}
tests := []struct {
mediaType domain.MediaType
wantIDs []string
}{
{domain.MediaTypeAudio, []string{"2", "5"}},
{domain.MediaTypeVideo, []string{"1", "4"}},
{domain.MediaTypeVideoAudio, []string{"3"}},
}
for _, tt := range tests {
got := FilterFormatsByType(formats, tt.mediaType)
if len(got) != len(tt.wantIDs) {
t.Errorf("FilterFormatsByType(%v) = %d results, want %d", tt.mediaType, len(got), len(tt.wantIDs))
continue
}
for i, f := range got {
if f.ID != tt.wantIDs[i] {
t.Errorf("FilterFormatsByType(%v)[%d].ID = %s, want %s", tt.mediaType, i, f.ID, tt.wantIDs[i])
}
}
}
}
func TestDeduplicateResolutions(t *testing.T) {
formats := []domain.Format{
{ID: "1", Resolution: "1920x1080", Height: 1080, IsVideoOnly: true},
{ID: "2", Resolution: "1920x1080", Height: 1080, IsVideoOnly: true},
{ID: "3", Resolution: "1280x720", Height: 720, IsVideoOnly: true},
{ID: "4", Bitrate: "128k", IsAudioOnly: true},
{ID: "5", Bitrate: "128k", IsAudioOnly: true},
}
got := DeduplicateResolutions(formats)
if len(got) != 3 {
t.Errorf("DeduplicateResolutions = %d results, want 3", len(got))
}
}
func TestContainerOptions(t *testing.T) {
tests := []struct {
mt domain.MediaType
want []string
}{
{domain.MediaTypeAudio, []string{"mp3", "m4a", "opus"}},
{domain.MediaTypeVideo, []string{"mp4", "mkv", "webm"}},
{domain.MediaTypeVideoAudio, []string{"mp4", "mkv"}},
}
for _, tt := range tests {
got := ContainerOptions(tt.mt)
if len(got) != len(tt.want) {
t.Errorf("ContainerOptions(%v) = %v, want %v", tt.mt, got, tt.want)
continue
}
for i, c := range got {
if c != tt.want[i] {
t.Errorf("ContainerOptions(%v)[%d] = %s, want %s", tt.mt, i, c, tt.want[i])
}
}
}
}
func TestContainerOptionsZeroValue(t *testing.T) {
got := ContainerOptions(domain.MediaType(0))
want := []string{"mp4"}
if len(got) != 1 || got[0] != want[0] {
t.Errorf("ContainerOptions(0) = %v, want %v", got, want)
}
}
func TestFormatLabel(t *testing.T) {
tests := []struct {
f domain.Format
want string
}{
{domain.Format{IsAudioOnly: true, Bitrate: "128k"}, "128k"},
{domain.Format{IsAudioOnly: true}, "audio"},
{domain.Format{Resolution: "1920x1080", Height: 1080}, "1920x1080"},
{domain.Format{Height: 720}, "720p"},
{domain.Format{ID: "137"}, "137"},
}
for _, tt := range tests {
got := formatLabel(tt.f)
if got != tt.want {
t.Errorf("formatLabel(%+v) = %q, want %q", tt.f, got, tt.want)
}
}
}