Use some code from ytnotifier

master
blank X 1 year ago
parent 1fd5ccaeb5
commit ae4b496a69
Signed by: blankie
GPG Key ID: CC15FC822C7F61F5
  1. 28
      autoytarchive/utils.py
  2. 76
      autoytarchive/workers.py
  3. 12
      youtube-dl-injected.py

@ -1,11 +1,29 @@
import os
import sys
import json
import shlex
import asyncio
from io import BytesIO
from youtube_dl.extractor import youtube
from . import config, client, seen_videos
youtube._try_get = _try_get = youtube.try_get
def traverse_dict(src):
for (key, value) in src.items():
if key == 'scheduledStartTime':
return value
if isinstance(value, dict):
if value := traverse_dict(value):
return value
return None
def try_get(src, getter, expected_type=None):
if reason := src.get('reason'):
if isinstance(reason, str) and (reason.startswith('This live event will begin in ') or reason.startswith('Premieres in ')):
if t := _try_get(src, traverse_dict, str):
src['reason'] = f'autoytarchive:{t} {reason}'
return _try_get(src, getter, expected_type)
youtube.try_get = try_get
async def split_files(filename, destination_dir):
args = [
'split',
@ -25,11 +43,3 @@ async def update_seen_videos():
file.name = 'autoytarchive.json'
file.seek(0)
await client.edit_message(config['config']['storage_chat_id'], config['config']['storage_message_id'], file=file)
async def get_video(video_id):
proc = await asyncio.create_subprocess_exec(sys.executable, 'youtube-dl-injected.py', '--dump-single-json', '--', video_id, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
stdout, stderr = await proc.communicate()
if proc.returncode:
error_message = next(i.strip() for i in stderr.decode().split('\n') if i.startswith('ERROR: '))
return error_message or (proc.returncode, (stderr + stdout).decode())
return json.loads(stdout)

@ -1,3 +1,4 @@
import re
import os
import json
import time
@ -9,10 +10,15 @@ import feedparser
from io import BytesIO
from decimal import Decimal
from urllib.parse import quote as urlencode, urlparse
from youtube_dl import YoutubeDL
from . import session, config, client, seen_videos
from .utils import split_files, update_seen_videos, get_video
from .utils import split_files, update_seen_videos
tmp_handled = []
live_regex = re.compile(r'error: (?:autoytarchive:([0-9]+) )?(?:this live event will begin|premieres) in .+', re.I)
strip_date = re.compile(r' \d{4}-\d{2}-\d{2} \d{2}:\d{2}$')
ytdl = YoutubeDL({'skip_download': True, 'no_color': True})
ytdl.add_default_info_extractors()
async def check_channels(nodl):
if nodl:
@ -65,30 +71,28 @@ async def check_video(video):
async def _check_video(video):
logging.info('Checking video %s', video['yt_videoid'])
first_try_live = waited = False
tmr_attempts = 1
too_many_requests_count = 1
while True:
video_json = await get_video(video['link'])
if isinstance(video_json, dict):
try:
video_json = await client.loop.run_in_executor(None, ytdl.extract_info, video['link'])
except BaseException as e:
wait_time = 30
message = str(e)
if '429' in message or 'too many' in message.lower():
wait_time = too_many_requests_count * 60 * 60
too_many_requests_count += 1
elif match := live_regex.match(message.rstrip('.')):
end_schedule_time = match.group(1) or 0
if end_schedule_time := int(end_schedule_time):
tmp_wait_time = end_schedule_time - time.time()
if tmp_wait_time > wait_time:
wait_time = tmp_wait_time
await asyncio.sleep(wait_time)
waited = True
else:
if not waited:
first_try_live = True
break
wait_time = 30
if isinstance(video_json, str):
error_message = video_json[7:]
if 'too many request' in error_message.lower():
wait_time = tmr_attempts * 60 * 60
tmr_attempts += 1
elif error_message.startswith('AUTOYTARCHIVE:'):
tmp = error_message.split(':', 1)[1].split(' ', 1)[0]
if tmp.isnumeric():
new_wait_time = int(tmp) - int(time.time())
if new_wait_time > 0:
wait_time = new_wait_time
logging.error('Error on video %s: %s', video['yt_videoid'], error_message)
else:
logging.error('Video %s returned status code %s\n%s', video['yt_videoid'], *video_json)
await asyncio.sleep(wait_time)
waited = True
if not video_json.get('is_live'):
first_try_live = False
video_queue.put_nowait((video_json, time.time(), first_try_live))
@ -119,17 +123,16 @@ async def _video_worker():
tempdir = tempdir_obj.name
if late_to_queue:
for i in range(5):
tmp = await get_video(video_json['id'])
if isinstance(tmp, dict):
video_json = tmp
break
wait_time = 30
if isinstance(tmp, str):
if 'too many request' in tmp.lower():
try:
tmp = await client.loop.run_in_executor(None, ytdl.extract_info, video_json['id'])
except BaseException as e:
e = str(e)
if '429' in e or 'too many request' in e.lower():
wait_time = (i + 1) * 60 * 60
logging.error('Error on video %s: %s', video_json['id'], tmp)
else:
logging.error('Video %s returned status code %s\n%s', video_json['id'], *video_json)
video_json = tmp
break
await asyncio.sleep(wait_time)
if video_json.get('requested_formats'):
for i in video_json['requested_formats']:
@ -180,16 +183,15 @@ async def _video_worker():
except BaseException:
logging.exception('Exception encountered when sending message to Telegram about download failure exception')
for i in range(5):
tmp = await get_video(video_json['id'])
if isinstance(tmp, dict):
video_json = tmp
break
if isinstance(tmp, str):
if 'too many request' in tmp.lower():
try:
tmp = await client.loop.run_in_executor(None, ytdl.extract_info, video_json['id'])
except BaseException as e:
e = str(e)
if '429' in e or 'too many request' in e.lower():
wait_time = (i + 1) * 60 * 60
logging.error('Error on video %s: %s', video_json['id'], tmp)
else:
logging.error('Video %s returned status code %s\n%s', video_json['id'], *tmp)
video_json = tmp
break
await asyncio.sleep(wait_time)
except BaseException:
tempdir_obj.cleanup()

@ -1,12 +0,0 @@
from youtube_dl.extractor import youtube
from youtube_dl import main
youtube._try_get = _try_get = youtube.try_get
def try_get(src, getter, expected_type=None):
res = _try_get(src, getter, expected_type)
if isinstance(res, str) and res.startswith('This live event will begin in '):
t = _try_get(src, lambda x: x['playabilityStatus']['liveStreamability']['liveStreamabilityRenderer']['offlineSlate']['liveStreamOfflineSlateRenderer']['scheduledStartTime'], str)
res = f'AUTOYTARCHIVE:{t} {res}'
return res
youtube.try_get = try_get
main()
Loading…
Cancel
Save