commit bf9c83f3df557465310d3a3e67d3365a0398a3da Author: blank X Date: Thu Mar 4 14:23:03 2021 +0700 Initial commit diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5efeb79 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +config.yaml +ytnotifier.session +ytnotifier.data diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..211185a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 blank X + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/example-config.yaml b/example-config.yaml new file mode 100644 index 0000000..aeaa176 --- /dev/null +++ b/example-config.yaml @@ -0,0 +1,14 @@ +telegram: + api_id: 0 + api_hash: https://my.telegram.org + bot_token: https://t.me/BotFather +storage: + # Useful for temporary file systems, can delete if you want + storage_chat_id: -1001222674489 + storage_message_id: 1377 +config: + notify_chat: 558772678 + wait_seconds: 1800 + channels: + - UCL_qhgtOy0dy1Agp8vkySQg + - UCHsx4Hqa-1ORjQTh9TYDhww diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..223c944 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +youtube-dl +feedparser +telethon +aiohttp diff --git a/ytnotifier.py b/ytnotifier.py new file mode 100644 index 0000000..a1ee9c6 --- /dev/null +++ b/ytnotifier.py @@ -0,0 +1,165 @@ +import re +import sys +import time +import html +import asyncio +import logging +import yaml +import feedparser +from telethon import TelegramClient +from youtube_dl import YoutubeDL +from youtube_dl.extractor import youtube +from aiohttp import ClientSession + +logging.basicConfig(level=logging.INFO) + +loadmode = False +if len(sys.argv) > 1: + loadmode = sys.argv[1] == 'loadmode' + +config = yaml.safe_load(open('config.yaml')) +api_id = config['telegram']['api_id'] +api_hash = config['telegram']['api_hash'] +bot_token = config['telegram'].get('bot_token') +storage_chat_id = storage_message_id = None +if config.get('storage'): + storage_chat_id = config['storage'].get('storage_chat_id') + storage_message_id = config['storage'].get('storage_message_id') +notify_chat = config['config']['notify_chat'] +wait_seconds = config['config']['wait_seconds'] +channels = config['config']['channels'] + +live_regex = re.compile(r'error: (?:ytnotifier:([0-9]+) )?this live event will begin in (.+)', re.I) +strip_date = re.compile(r' \d{4}-\d{2}-\d{2} \d{2}:\d{2}$') +ytdl = YoutubeDL({'skip_download': True, 'no_color': True}) +ytdl.add_default_info_extractors() +client = TelegramClient('ytnotifier', api_id, api_hash) +client.parse_mode = 'html' +seen_videos = set() +seen_videos_lock = asyncio.Lock() +tmp_handled_videos = set() +async def save_seen_videos(): + async with seen_videos_lock: + with open('ytnotifier.data', 'w') as file: + file.write('0\n') + for i in seen_videos: + file.write(f'{i}\n') + if storage_chat_id and storage_message_id: + await client.edit_message(storage_chat_id, storage_message_id, file='ytnotifier.data') + +youtube._try_get = _try_get = youtube.try_get +def traverse_dict(src): + for (key, value) in src.items(): + if key == 'scheduledStartTime': + return value + if isinstance(value, dict): + if value := traverse_dict(value): + return value + return None + +def try_get(src, getter, expected_type=None): + if reason := src.get('reason'): + if isinstance(reason, str) and reason.startswith('This live event will begin in '): + t = _try_get(src, traverse_dict, str) + if t: + src['reason'] = f'ytnotifier:{t} {reason}' + return _try_get(src, getter, expected_type) +youtube.try_get = try_get + +async def _handle_video(video_id, video_title): + last_was_few_moments = False + too_many_attempts_count = 1 + video_url = f'https://www.youtube.com/watch?v={video_id}' + notify_text = 'New video' + while True: + try: + video_data = await client.loop.run_in_executor(None, ytdl.extract_info, video_url) + except BaseException as e: + wait_time = 30 + message = str(e) + if '429' in message or 'too many' in message.lower(): + wait_time = too_many_attempts_count * 60 * 60 + too_many_attempts_count += 1 + elif match := live_regex.match(message.rstrip('.')): + notify_text = 'Live event started' + end_schedule_time = match.group(1) or 0 + human_end_schedule_time = match.group(2) + if end_schedule_time := int(end_schedule_time): + tmp_wait_time = end_schedule_time - time.time() + if tmp_wait_time > wait_time: + wait_time = tmp_wait_time + await client.send_message(notify_chat, f'Live event starting in {human_end_schedule_time}: {html.escape(video_title)}') + elif not last_was_few_moments: + await client.send_message(notify_chat, f'Live event starting in {human_end_schedule_time}: {html.escape(video_title)}') + elif not last_was_few_moments: + await client.send_message(notify_chat, f'Live event starting in {human_end_schedule_time}: {html.escape(video_title)}') + last_was_few_moments = 'moment' in human_end_schedule_time.lower() + await asyncio.sleep(wait_time) + else: + if video_data.get('is_live'): + notify_text = 'Live event started' + break + if tmp_video_title := video_data.get('title'): + video_title = strip_date.sub('', tmp_video_title) + await client.send_message(notify_chat, f'{notify_text}: {html.escape(video_title)}') + async with seen_videos_lock: + seen_videos.add(video_id) + +async def handle_video(video_id, video_title): + try: + while True: + try: + await _handle_video(video_id, video_title) + except BaseException: + logging.exception('Exception raised while notifying %s (%s)', video_id, video_title) + else: + await save_seen_videos() + break + finally: + tmp_handled_videos.discard(video_id) + +async def main(): + await client.start(bot_token=bot_token) + if storage_chat_id and storage_message_id: + try: + m = await client.get_messages(storage_chat_id, ids=storage_message_id) + await m.download_media('ytnotifier.data') + except BaseException: + logging.exception('Exception raised when downloading ytnotifier.data') + try: + with open('ytnotifier.data') as file: + version = file.readline().strip() + if version != '0': + logging.error('ytnotifier.data has incompatible version %s', version) + else: + while True: + video_id = file.readline().strip() + if not video_id: + break + seen_videos.add(video_id) + except BaseException: + logging.exception('Exception raised when parsing ytnotifier.data') + + async with ClientSession() as session: + while True: + for i in channels: + logging.info('Checking %s', i) + async with session.get(f'https://www.youtube.com/feeds/videos.xml?channel_id={i}&a={time.time()}', headers={'Cache-Control': 'no-store, max-age=0'}) as resp: + d = feedparser.parse(await resp.text()) + for e in d['entries']: + video_id = e['yt_videoid'] + video_title = e['title'] + if video_id not in seen_videos and video_id not in tmp_handled_videos: + tmp_handled_videos.add(video_id) + if loadmode: + seen_videos.add(video_id) + else: + logging.info('Handling %s', video_id) + asyncio.create_task(handle_video(video_id, video_title)) + if loadmode: + await save_seen_videos() + break + await asyncio.sleep(wait_seconds) + +if __name__ == '__main__': + client.loop.run_until_complete(main())