fix: enforce max_concurrent and global speed-limit distribution

This commit is contained in:
2026-03-03 18:36:56 +09:00
parent 1cdf68cc59
commit 7a087ce9c5
5 changed files with 163 additions and 4 deletions

View File

@@ -47,6 +47,8 @@ class FfmpegHlsDownloader(BaseDownloader):
# ffmpeg 명령어 구성
ffmpeg_path = options.get('ffmpeg_path', 'ffmpeg')
if options.get('effective_max_download_rate') or options.get('max_download_rate'):
logger.warning('[GDM] ffmpeg_hls downloader does not support strict bandwidth cap; total limit may be approximate for HLS tasks.')
cmd = [ffmpeg_path, '-y']

View File

@@ -5,6 +5,8 @@ HTTP 직접 다운로더
"""
import os
import traceback
import re
import time
from typing import Dict, Any, Optional, Callable
from .base import BaseDownloader
@@ -19,6 +21,21 @@ except:
class HttpDirectDownloader(BaseDownloader):
"""HTTP 직접 다운로더"""
@staticmethod
def _rate_to_bps(rate_value: Any) -> float:
if rate_value is None:
return 0.0
value = str(rate_value).strip().upper()
if not value or value in ('0', 'UNLIMITED'):
return 0.0
m = re.match(r'^(\d+(?:\.\d+)?)\s*([KMG])(?:I?B)?$', value)
if not m:
return 0.0
num = float(m.group(1))
unit = m.group(2)
mul = {'K': 1024, 'M': 1024 ** 2, 'G': 1024 ** 3}[unit]
return num * mul
def download(
self,
@@ -53,6 +70,9 @@ class HttpDirectDownloader(BaseDownloader):
total_size = int(response.headers.get('content-length', 0))
downloaded = 0
chunk_size = 1024 * 1024 # 1MB 청크
max_rate = options.get('effective_max_download_rate') or options.get('max_download_rate')
rate_bps = self._rate_to_bps(max_rate)
start_time = time.monotonic()
with open(filepath, 'wb') as f:
for chunk in response.iter_content(chunk_size=chunk_size):
@@ -62,6 +82,13 @@ class HttpDirectDownloader(BaseDownloader):
if chunk:
f.write(chunk)
downloaded += len(chunk)
# 평균 다운로드 속도를 제한(총량 제한 분배값 포함)
if rate_bps > 0:
elapsed = max(0.001, time.monotonic() - start_time)
expected_elapsed = downloaded / rate_bps
if expected_elapsed > elapsed:
time.sleep(expected_elapsed - elapsed)
if total_size > 0 and progress_callback:
progress = int(downloaded / total_size * 100)

View File

@@ -26,6 +26,19 @@ class YtdlpAria2Downloader(BaseDownloader):
def __init__(self):
super().__init__()
self._process: Optional[subprocess.Popen] = None
@staticmethod
def _normalize_rate(raw_rate: Any) -> str:
"""속도 제한 문자열 정규화 (예: 6MB -> 6M, 0/None -> '')"""
if raw_rate is None:
return ''
value = str(raw_rate).strip().upper()
if not value or value in ('0', '0B', 'UNLIMITED'):
return ''
m = re.match(r'^(\d+(?:\.\d+)?)\s*([KMG])(?:I?B)?$', value)
if m:
return f'{m.group(1)}{m.group(2)}'
return value
def download(
self,
@@ -61,8 +74,12 @@ class YtdlpAria2Downloader(BaseDownloader):
cmd.extend(['--print', 'before_dl:GDM_FIX:thumb:%(thumbnail)s'])
# 속도 제한 설정
max_rate = P.ModelSetting.get('max_download_rate')
rate_limited = bool(max_rate and max_rate != '0')
max_rate = self._normalize_rate(
options.get('effective_max_download_rate')
or options.get('max_download_rate')
or P.ModelSetting.get('max_download_rate')
)
rate_limited = bool(max_rate)
# aria2c 사용 (설치되어 있으면)
aria2c_path = options.get('aria2c_path', 'aria2c')
@@ -83,6 +100,10 @@ class YtdlpAria2Downloader(BaseDownloader):
# yt-dlp native downloader 제한 (external-downloader 미사용/보조 경로)
if rate_limited:
cmd.extend(['--limit-rate', max_rate])
if options.get('is_global_rate_split'):
logger.info(f'[GDM] global split limit enabled: {max_rate}/s per task')
else:
logger.info(f'[GDM] download speed limit enabled: {max_rate}/s')
# 포맷 선택
format_spec = options.get('format')