blankie
/
autoytarchive
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Auto-kill when ffmpeg doesn't output anything for 5m

This commit is contained in:
blank X 2021-04-11 09:38:22 +07:00
parent d37ed659cd
commit 4a7ad768c7
Signed by: blankie
GPG Key ID: CC15FC822C7F61F5
1 changed files with 69 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

121
autoytarchive/workers.py
View File

@ -165,60 +165,77 @@ async def _video_worker():
command.extend(('-c', 'copy', video_filename)) command.extend(('-c', 'copy', video_filename))
return command return command
video_filename = os.path.join(tempdir, video_json['id'] + '.mkv') video_filename = os.path.join(tempdir, video_json['id'] + '.mkv')
proc = await asyncio.create_subprocess_exec(*(await construct_command())) with open(video_json['id'] + '.mkv', 'ab') as log_file:
text = 'New video' async def auto_kill(proc):
if is_late: last_tell = log_file.tell()
text += ' (is late)' while True:
video_json['title'] = strip_date.sub('', video_json['title']).strip() await asyncio.sleep(300)
text += f': {video_json["title"]}\nhttps://youtube.com/watch?v={video_json["id"]}' current_tell = log_file.tell()
with BytesIO(json.dumps(video_json, indent=4).encode()) as file: if current_tell == last_tell:
file.name = video_json['id'] + '.json' break
file.seek(0) last_tell = current_tell
await client.send_file(config['config']['storage_chat_id'], file, caption=text, parse_mode=None) proc.stdin.write(b'q')
if video_json.get('thumbnail'): await proc.stdin.drain()
thumbnail_ext = os.path.splitext(urlparse(video_json['thumbnail']).path)[1] proc = await asyncio.create_subprocess_exec(*(await construct_command()), stdin=asyncio.subprocess.PIPE, stdout=log_file, stderr=log_file)
thumbnail_filename = os.path.join(tempdir, video_json['id'] + thumbnail_ext) text = 'New video'
async with session.get(video_json['thumbnail']) as resp: if is_late:
with open(thumbnail_filename, 'wb') as file: text += ' (is late)'
while True: video_json['title'] = strip_date.sub('', video_json['title']).strip()
chunk = await resp.content.read(4096) text += f': {video_json["title"]}\nhttps://youtube.com/watch?v={video_json["id"]}'
if not chunk: with BytesIO(json.dumps(video_json, indent=4).encode()) as file:
break file.name = video_json['id'] + '.json'
file.write(chunk) file.seek(0)
await client.send_file(config['config']['storage_chat_id'], thumbnail_filename, caption=text, parse_mode=None) await client.send_file(config['config']['storage_chat_id'], file, caption=text, parse_mode=None)
os.remove(thumbnail_filename) if video_json.get('thumbnail'):
for _ in range(50): thumbnail_ext = os.path.splitext(urlparse(video_json['thumbnail']).path)[1]
await proc.communicate() thumbnail_filename = os.path.join(tempdir, video_json['id'] + thumbnail_ext)
if not proc.returncode: async with session.get(video_json['thumbnail']) as resp:
break with open(thumbnail_filename, 'wb') as file:
wait_time = 30 while True:
if video_json.get('duration'): chunk = await resp.content.read(4096)
is_manifest = False if not chunk:
if video_json.get('url'): break
is_manifest = urlparse(video_json['url']).netloc == 'manifest.googlevideo.com' file.write(chunk)
if not is_manifest and video_json.get('requested_formats'): await client.send_file(config['config']['storage_chat_id'], thumbnail_filename, caption=text, parse_mode=None)
for i in video_json['requested_formats']: os.remove(thumbnail_filename)
if urlparse(i['url']).netloc == 'manifest.googlevideo.com': for _ in range(50):
is_manifest = True auto_kill_task = asyncio.create_task(auto_kill(proc))
break await proc.communicate()
if is_manifest: auto_kill_task.cancel()
await asyncio.sleep(video_json['duration'] + 30)
try:
await client.send_message(config['config']['storage_chat_id'], f'Failed to download video {video_json["id"]}, please check logs', parse_mode=None)
except BaseException:
logging.exception('Exception encountered when sending message to Telegram about download failure exception')
for i in range(5):
try: try:
tmp = await client.loop.run_in_executor(None, ytdl.extract_info, video_json['id']) await auto_kill_task
except BaseException as e: except asyncio.CancelledError:
e = str(e) pass
if '429' in e or 'too many request' in e.lower(): if not proc.returncode:
wait_time = (i + 1) * 60 * 60
else:
video_json = tmp
break break
await asyncio.sleep(wait_time) wait_time = 30
proc = await asyncio.create_subprocess_exec(*(await construct_command())) if video_json.get('duration'):
is_manifest = False
if video_json.get('url'):
is_manifest = urlparse(video_json['url']).netloc == 'manifest.googlevideo.com'
if not is_manifest and video_json.get('requested_formats'):
for i in video_json['requested_formats']:
if urlparse(i['url']).netloc == 'manifest.googlevideo.com':
is_manifest = True
break
if is_manifest:
await asyncio.sleep(video_json['duration'] + 30)
try:
await client.send_message(config['config']['storage_chat_id'], f'Failed to download video {video_json["id"]}, please check logs', parse_mode=None)
except BaseException:
logging.exception('Exception encountered when sending message to Telegram about download failure exception')
for i in range(5):
try:
tmp = await client.loop.run_in_executor(None, ytdl.extract_info, video_json['id'])
except BaseException as e:
e = str(e)
if '429' in e or 'too many request' in e.lower():
wait_time = (i + 1) * 60 * 60
else:
video_json = tmp
break
await asyncio.sleep(wait_time)
proc = await asyncio.create_subprocess_exec(*(await construct_command()), stdin=asyncio.subprocess.PIPE, stdout=log_file, stderr=log_file)
except BaseException: except BaseException:
tempdir_obj.cleanup() tempdir_obj.cleanup()
raise raise