|
|
|
@ -1,3 +1,4 @@ |
|
|
|
|
import re |
|
|
|
|
import os |
|
|
|
|
import json |
|
|
|
|
import time |
|
|
|
@ -9,10 +10,15 @@ import feedparser |
|
|
|
|
from io import BytesIO |
|
|
|
|
from decimal import Decimal |
|
|
|
|
from urllib.parse import quote as urlencode, urlparse |
|
|
|
|
from youtube_dl import YoutubeDL |
|
|
|
|
from . import session, config, client, seen_videos |
|
|
|
|
from .utils import split_files, update_seen_videos, get_video |
|
|
|
|
from .utils import split_files, update_seen_videos |
|
|
|
|
|
|
|
|
|
tmp_handled = [] |
|
|
|
|
live_regex = re.compile(r'error: (?:autoytarchive:([0-9]+) )?(?:this live event will begin|premieres) in .+', re.I) |
|
|
|
|
strip_date = re.compile(r' \d{4}-\d{2}-\d{2} \d{2}:\d{2}$') |
|
|
|
|
ytdl = YoutubeDL({'skip_download': True, 'no_color': True}) |
|
|
|
|
ytdl.add_default_info_extractors() |
|
|
|
|
|
|
|
|
|
async def check_channels(nodl): |
|
|
|
|
if nodl: |
|
|
|
@ -65,30 +71,28 @@ async def check_video(video): |
|
|
|
|
async def _check_video(video): |
|
|
|
|
logging.info('Checking video %s', video['yt_videoid']) |
|
|
|
|
first_try_live = waited = False |
|
|
|
|
tmr_attempts = 1 |
|
|
|
|
too_many_requests_count = 1 |
|
|
|
|
while True: |
|
|
|
|
video_json = await get_video(video['link']) |
|
|
|
|
if isinstance(video_json, dict): |
|
|
|
|
try: |
|
|
|
|
video_json = await client.loop.run_in_executor(None, ytdl.extract_info, video['link']) |
|
|
|
|
except BaseException as e: |
|
|
|
|
wait_time = 30 |
|
|
|
|
message = str(e) |
|
|
|
|
if '429' in message or 'too many' in message.lower(): |
|
|
|
|
wait_time = too_many_requests_count * 60 * 60 |
|
|
|
|
too_many_requests_count += 1 |
|
|
|
|
elif match := live_regex.match(message.rstrip('.')): |
|
|
|
|
end_schedule_time = match.group(1) or 0 |
|
|
|
|
if end_schedule_time := int(end_schedule_time): |
|
|
|
|
tmp_wait_time = end_schedule_time - time.time() |
|
|
|
|
if tmp_wait_time > wait_time: |
|
|
|
|
wait_time = tmp_wait_time |
|
|
|
|
await asyncio.sleep(wait_time) |
|
|
|
|
waited = True |
|
|
|
|
else: |
|
|
|
|
if not waited: |
|
|
|
|
first_try_live = True |
|
|
|
|
break |
|
|
|
|
wait_time = 30 |
|
|
|
|
if isinstance(video_json, str): |
|
|
|
|
error_message = video_json[7:] |
|
|
|
|
if 'too many request' in error_message.lower(): |
|
|
|
|
wait_time = tmr_attempts * 60 * 60 |
|
|
|
|
tmr_attempts += 1 |
|
|
|
|
elif error_message.startswith('AUTOYTARCHIVE:'): |
|
|
|
|
tmp = error_message.split(':', 1)[1].split(' ', 1)[0] |
|
|
|
|
if tmp.isnumeric(): |
|
|
|
|
new_wait_time = int(tmp) - int(time.time()) |
|
|
|
|
if new_wait_time > 0: |
|
|
|
|
wait_time = new_wait_time |
|
|
|
|
logging.error('Error on video %s: %s', video['yt_videoid'], error_message) |
|
|
|
|
else: |
|
|
|
|
logging.error('Video %s returned status code %s\n%s', video['yt_videoid'], *video_json) |
|
|
|
|
await asyncio.sleep(wait_time) |
|
|
|
|
waited = True |
|
|
|
|
if not video_json.get('is_live'): |
|
|
|
|
first_try_live = False |
|
|
|
|
video_queue.put_nowait((video_json, time.time(), first_try_live)) |
|
|
|
@ -119,17 +123,16 @@ async def _video_worker(): |
|
|
|
|
tempdir = tempdir_obj.name |
|
|
|
|
if late_to_queue: |
|
|
|
|
for i in range(5): |
|
|
|
|
tmp = await get_video(video_json['id']) |
|
|
|
|
if isinstance(tmp, dict): |
|
|
|
|
video_json = tmp |
|
|
|
|
break |
|
|
|
|
wait_time = 30 |
|
|
|
|
if isinstance(tmp, str): |
|
|
|
|
if 'too many request' in tmp.lower(): |
|
|
|
|
try: |
|
|
|
|
tmp = await client.loop.run_in_executor(None, ytdl.extract_info, video_json['id']) |
|
|
|
|
except BaseException as e: |
|
|
|
|
e = str(e) |
|
|
|
|
if '429' in e or 'too many request' in e.lower(): |
|
|
|
|
wait_time = (i + 1) * 60 * 60 |
|
|
|
|
logging.error('Error on video %s: %s', video_json['id'], tmp) |
|
|
|
|
else: |
|
|
|
|
logging.error('Video %s returned status code %s\n%s', video_json['id'], *video_json) |
|
|
|
|
video_json = tmp |
|
|
|
|
break |
|
|
|
|
await asyncio.sleep(wait_time) |
|
|
|
|
if video_json.get('requested_formats'): |
|
|
|
|
for i in video_json['requested_formats']: |
|
|
|
@ -180,16 +183,15 @@ async def _video_worker(): |
|
|
|
|
except BaseException: |
|
|
|
|
logging.exception('Exception encountered when sending message to Telegram about download failure exception') |
|
|
|
|
for i in range(5): |
|
|
|
|
tmp = await get_video(video_json['id']) |
|
|
|
|
if isinstance(tmp, dict): |
|
|
|
|
video_json = tmp |
|
|
|
|
break |
|
|
|
|
if isinstance(tmp, str): |
|
|
|
|
if 'too many request' in tmp.lower(): |
|
|
|
|
try: |
|
|
|
|
tmp = await client.loop.run_in_executor(None, ytdl.extract_info, video_json['id']) |
|
|
|
|
except BaseException as e: |
|
|
|
|
e = str(e) |
|
|
|
|
if '429' in e or 'too many request' in e.lower(): |
|
|
|
|
wait_time = (i + 1) * 60 * 60 |
|
|
|
|
logging.error('Error on video %s: %s', video_json['id'], tmp) |
|
|
|
|
else: |
|
|
|
|
logging.error('Video %s returned status code %s\n%s', video_json['id'], *tmp) |
|
|
|
|
video_json = tmp |
|
|
|
|
break |
|
|
|
|
await asyncio.sleep(wait_time) |
|
|
|
|
except BaseException: |
|
|
|
|
tempdir_obj.cleanup() |
|
|
|
|