fix: enforce max_concurrent and global speed-limit distribution
This commit is contained in:
@@ -47,6 +47,8 @@ class FfmpegHlsDownloader(BaseDownloader):
|
||||
|
||||
# ffmpeg 명령어 구성
|
||||
ffmpeg_path = options.get('ffmpeg_path', 'ffmpeg')
|
||||
if options.get('effective_max_download_rate') or options.get('max_download_rate'):
|
||||
logger.warning('[GDM] ffmpeg_hls downloader does not support strict bandwidth cap; total limit may be approximate for HLS tasks.')
|
||||
|
||||
cmd = [ffmpeg_path, '-y']
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ HTTP 직접 다운로더
|
||||
"""
|
||||
import os
|
||||
import traceback
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
|
||||
from .base import BaseDownloader
|
||||
@@ -19,6 +21,21 @@ except:
|
||||
|
||||
class HttpDirectDownloader(BaseDownloader):
|
||||
"""HTTP 직접 다운로더"""
|
||||
|
||||
@staticmethod
|
||||
def _rate_to_bps(rate_value: Any) -> float:
|
||||
if rate_value is None:
|
||||
return 0.0
|
||||
value = str(rate_value).strip().upper()
|
||||
if not value or value in ('0', 'UNLIMITED'):
|
||||
return 0.0
|
||||
m = re.match(r'^(\d+(?:\.\d+)?)\s*([KMG])(?:I?B)?$', value)
|
||||
if not m:
|
||||
return 0.0
|
||||
num = float(m.group(1))
|
||||
unit = m.group(2)
|
||||
mul = {'K': 1024, 'M': 1024 ** 2, 'G': 1024 ** 3}[unit]
|
||||
return num * mul
|
||||
|
||||
def download(
|
||||
self,
|
||||
@@ -53,6 +70,9 @@ class HttpDirectDownloader(BaseDownloader):
|
||||
total_size = int(response.headers.get('content-length', 0))
|
||||
downloaded = 0
|
||||
chunk_size = 1024 * 1024 # 1MB 청크
|
||||
max_rate = options.get('effective_max_download_rate') or options.get('max_download_rate')
|
||||
rate_bps = self._rate_to_bps(max_rate)
|
||||
start_time = time.monotonic()
|
||||
|
||||
with open(filepath, 'wb') as f:
|
||||
for chunk in response.iter_content(chunk_size=chunk_size):
|
||||
@@ -62,6 +82,13 @@ class HttpDirectDownloader(BaseDownloader):
|
||||
if chunk:
|
||||
f.write(chunk)
|
||||
downloaded += len(chunk)
|
||||
|
||||
# 평균 다운로드 속도를 제한(총량 제한 분배값 포함)
|
||||
if rate_bps > 0:
|
||||
elapsed = max(0.001, time.monotonic() - start_time)
|
||||
expected_elapsed = downloaded / rate_bps
|
||||
if expected_elapsed > elapsed:
|
||||
time.sleep(expected_elapsed - elapsed)
|
||||
|
||||
if total_size > 0 and progress_callback:
|
||||
progress = int(downloaded / total_size * 100)
|
||||
|
||||
@@ -26,6 +26,19 @@ class YtdlpAria2Downloader(BaseDownloader):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self._process: Optional[subprocess.Popen] = None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_rate(raw_rate: Any) -> str:
|
||||
"""속도 제한 문자열 정규화 (예: 6MB -> 6M, 0/None -> '')"""
|
||||
if raw_rate is None:
|
||||
return ''
|
||||
value = str(raw_rate).strip().upper()
|
||||
if not value or value in ('0', '0B', 'UNLIMITED'):
|
||||
return ''
|
||||
m = re.match(r'^(\d+(?:\.\d+)?)\s*([KMG])(?:I?B)?$', value)
|
||||
if m:
|
||||
return f'{m.group(1)}{m.group(2)}'
|
||||
return value
|
||||
|
||||
def download(
|
||||
self,
|
||||
@@ -61,8 +74,12 @@ class YtdlpAria2Downloader(BaseDownloader):
|
||||
cmd.extend(['--print', 'before_dl:GDM_FIX:thumb:%(thumbnail)s'])
|
||||
|
||||
# 속도 제한 설정
|
||||
max_rate = P.ModelSetting.get('max_download_rate')
|
||||
rate_limited = bool(max_rate and max_rate != '0')
|
||||
max_rate = self._normalize_rate(
|
||||
options.get('effective_max_download_rate')
|
||||
or options.get('max_download_rate')
|
||||
or P.ModelSetting.get('max_download_rate')
|
||||
)
|
||||
rate_limited = bool(max_rate)
|
||||
|
||||
# aria2c 사용 (설치되어 있으면)
|
||||
aria2c_path = options.get('aria2c_path', 'aria2c')
|
||||
@@ -83,6 +100,10 @@ class YtdlpAria2Downloader(BaseDownloader):
|
||||
# yt-dlp native downloader 제한 (external-downloader 미사용/보조 경로)
|
||||
if rate_limited:
|
||||
cmd.extend(['--limit-rate', max_rate])
|
||||
if options.get('is_global_rate_split'):
|
||||
logger.info(f'[GDM] global split limit enabled: {max_rate}/s per task')
|
||||
else:
|
||||
logger.info(f'[GDM] download speed limit enabled: {max_rate}/s')
|
||||
|
||||
# 포맷 선택
|
||||
format_spec = options.get('format')
|
||||
|
||||
Reference in New Issue
Block a user