MyAnimeBot-DEV/myanimebot.py

#!/usr/bin/env python3
# Copyright Penta & lulu (c) 2018/2021 - Under BSD License - Based on feed2discord.py by Eric Eisenhart

# Compatible for Python 3.7.X

# Library import
import asyncio
import logging
import sys
import urllib.request
from configparser import ConfigParser
from datetime import datetime
from typing import List, Tuple

import aiohttp
import discord
import feedparser
from aiohttp.web_exceptions import HTTPError, HTTPNotModified
from dateutil.parser import parse as parse_datetime
from html2text import HTML2Text

# Our modules
import myanimebot.anilist as anilist
import myanimebot.globals as globals
import myanimebot.utils as utils
import myanimebot.myanimelist as myanimelist
from myanimebot.discord import send_embed_wrapper, build_embed


if not sys.version_info[:2] >= (3, 7):
	print("ERROR: Requires python 3.7 or newer.")
	exit(1)

	
# Main function that check the RSS feeds from MyAnimeList
async def background_check_feed(asyncioloop):
	globals.logger.info("Starting up background_check_feed")
	
	# We configure the http header
	http_headers = { "User-Agent": "MyAnimeBot Discord Bot v" + globals.VERSION, }
	
	await globals.client.wait_until_ready()
	
	globals.logger.debug("Discord client connected, unlocking background_check_feed...")
	
	while not globals.client.is_closed():
		try:
			db_user = globals.conn.cursor(buffered=True, dictionary=True)
			db_user.execute("SELECT mal_user, servers FROM t_users WHERE service=%s", [globals.SERVICE_MAL])
			data_user = db_user.fetchone()
		except Exception as e:
			globals.logger.critical("Database unavailable! (" + str(e) + ")")
			quit()

		while data_user is not None:
			user = utils.User(id=None,
								service_id=None,
								name=data_user[globals.DB_USER_NAME],
								servers=data_user["servers"].split(','))
			stop_boucle = 0
			feed_type = 1

			try:
				while stop_boucle == 0 :
					try:
						async with aiohttp.ClientSession() as httpclient:
							if feed_type == 1 :
								http_response = await httpclient.request("GET", "https://myanimelist.net/rss.php?type=rm&u=" + user.name, headers=http_headers)
								media = "manga"
							else : 
								http_response = await httpclient.request("GET", "https://myanimelist.net/rss.php?type=rw&u=" + user.name, headers=http_headers)
								media = "anime"
					except Exception as e:
						globals.logger.error("Error while loading RSS (" + str(feed_type) + ") of '" + user.name + "': " + str(e))
						break

					http_data = await http_response.read()
					feeds_data = feedparser.parse(http_data)
					
					for feed_data in feeds_data.entries:
						pubDateRaw = datetime.strptime(feed_data.published, '%a, %d %b %Y %H:%M:%S %z').astimezone(globals.timezone)
						pubDate = pubDateRaw.strftime("%Y-%m-%d %H:%M:%S")
						if feed_type == 1:
							media_type = utils.MediaType.MANGA
						else:
							media_type = utils.MediaType.ANIME

						feed = myanimelist.build_feed_from_data(feed_data, user, None, pubDateRaw.timestamp(), media_type)
						
						cursor = globals.conn.cursor(buffered=True)
						cursor.execute("SELECT published, title, url, type FROM t_feeds WHERE published=%s AND title=%s AND user=%s AND type=%s AND obsolete=0 AND service=%s", [pubDate, feed.media.name, user.name, feed.get_status_str(), globals.SERVICE_MAL])
						data = cursor.fetchone()

						if data is None:
							var = datetime.now(globals.timezone) - pubDateRaw
							
							globals.logger.debug(" - " + feed.media.name + ": " + str(var.total_seconds()))
						
							if var.total_seconds() < globals.secondMax:
								globals.logger.info(user.name + ": Item '" + feed.media.name + "' not seen, processing...")
								
								cursor.execute("SELECT thumbnail FROM t_animes WHERE guid=%s AND service=%s LIMIT 1", [feed.media.url, globals.SERVICE_MAL]) # TODO Change that ?
								data_img = cursor.fetchone()
								
								if data_img is None:
									try:
										image = myanimelist.get_thumbnail(feed.media.url)
										
										globals.logger.info("First time seeing this " + media + ", adding thumbnail into database: " + image)
									except Exception as e:
										globals.logger.warning("Error while getting the thumbnail: " + str(e))
										image = ""
										
									cursor.execute("INSERT INTO t_animes (guid, service, title, thumbnail, found, discoverer, media) VALUES (%s, %s, %s, %s, NOW(), %s, %s)", [feed.media.url, globals.SERVICE_MAL, feed.media.name, image, user.name, media])
									globals.conn.commit()
								else: image = data_img[0]
								feed.media.image = image

								cursor.execute("UPDATE t_feeds SET obsolete=1 WHERE published=%s AND title=%s AND user=%s AND service=%s", [pubDate, feed.media.name, user.name, globals.SERVICE_MAL])
								cursor.execute("INSERT INTO t_feeds (published, title, service, url, user, found, type) VALUES (%s, %s, %s, %s, %s, NOW(), %s)", (pubDate, feed.media.name, globals.SERVICE_MAL, feed.media.url, user.name, feed.get_status_str()))
								globals.conn.commit()
								
								for server in user.servers:
									db_srv = globals.conn.cursor(buffered=True)
									db_srv.execute("SELECT channel FROM t_servers WHERE server = %s", [server])
									data_channel = db_srv.fetchone()
									
									while data_channel is not None:
										for channel in data_channel: await send_embed_wrapper(asyncioloop, channel, globals.client, build_embed(feed))
										
										data_channel = db_srv.fetchone()
					if feed_type == 1:
						feed_type = 0
						await asyncio.sleep(globals.MYANIMELIST_SECONDS_BETWEEN_REQUESTS)
					else:
						stop_boucle = 1
					
			except Exception as e:
				globals.logger.exception("Error when parsing RSS for '" + user.name + "': \n")
			
			await asyncio.sleep(globals.MYANIMELIST_SECONDS_BETWEEN_REQUESTS)

			data_user = db_user.fetchone()


async def fetch_activities_anilist():
	await anilist.check_new_activities()


@globals.client.event
async def on_ready():
	globals.logger.info("Logged in as " + globals.client.user.name + " (" + str(globals.client.user.id) + ")")

	globals.logger.info("Starting all tasks...")

	globals.task_feed = globals.client.loop.create_task(background_check_feed(globals.client.loop))
	globals.task_feed_anilist = globals.client.loop.create_task(anilist.background_check_feed(globals.client.loop))
	globals.task_thumbnail = globals.client.loop.create_task(update_thumbnail_catalog(globals.client.loop))
	globals.task_gameplayed = globals.client.loop.create_task(change_gameplayed(globals.client.loop))


@globals.client.event
async def on_error(event, *args, **kwargs):
    globals.logger.exception("Crap! An unknown Discord error occured...")


def build_info_cmd_message(users, server, channels, filters : List[utils.Service]) -> str:
	''' Build the corresponding message for the info command '''

	registered_channel = globals.client.get_channel(int(channels[0]["channel"]))

	# Store users
	mal_users = []
	anilist_users = []
	for user in users:
		# If user is part of the server, add it to the message
		if str(server.id) in user['servers'].split(','):
			try:
				user_service = utils.Service.from_str(user["service"])
				if user_service == utils.Service.MAL:
					mal_users.append(user[globals.DB_USER_NAME])
				elif user_service == utils.Service.ANILIST:
					anilist_users.append(user[globals.DB_USER_NAME])
			except NotImplementedError:
				pass # Nothing to do here

	if not mal_users and not anilist_users:
		return "No users registered on this server. Try to add one."
	else:
		message =  'Registered user(s) on **{}**\n\n'.format(server)
		if mal_users: # If not empty
			# Don't print if there is filters and MAL is not in them
			if not filters or (filters and utils.Service.MAL in filters): 
				message += '**MyAnimeList** users:\n'
				message += '```{}```\n'.format(', '.join(mal_users))
		if anilist_users: # If not empty
			# Don't print if there is filters and MAL is not in them
			if not filters or (filters and utils.Service.ANILIST in filters):
				message += '**AniList** users:\n'
				message += '```{}```\n'.format(', '.join(anilist_users))
		message += 'Assigned channel : **{}**'.format(registered_channel)
	return message


def get_service_filters_list(filters : str) -> List[utils.Service]:
	''' Creates and returns a service filter list from a comma-separated string '''

	filters_list = []
	for filter in filters.split(','):
		try:
			filters_list.append(utils.Service.from_str(filter))
		except NotImplementedError:
			pass # Ignore incorrect filter
	return filters_list


async def info_cmd(message, words):
	''' Processes the command "info" and sends a message '''

	# Get filters if available
	filters = []
	if (len(words) >= 3): # If filters are specified
		filters = get_service_filters_list(words[2])

	server = message.guild
	if utils.is_server_in_db(server.id) == False:
		 await message.channel.send("The server **{}** is not in our database.".format(server))
	else:
		users = utils.get_users()
		channels = utils.get_channels(server.id)
		if channels is None:
			await message.channel.send("No channel assigned for this bot on this server.")
		else:
			await message.channel.send(build_info_cmd_message(users, server, channels, filters))


def check_user_name_validity(user_name: str, service : utils.Service) -> Tuple[bool, str]:
	""" Check if user_name exists on a specific service.
		
		Returns:
			- bool: 	True if user_name exists
			- str:		Error string if the user does not exist
	"""

	if service == utils.Service.MAL:
		try:
			# Ping user profile to check validity
			urllib.request.urlopen('{}{}'.format(globals.MAL_PROFILE_URL, user_name))
		except urllib.error.HTTPError as e:
			if (e.code == 404): # URL profile not found
				return False, "User **{}** doesn't exist on MyAnimeList!".format(user_name)
			else:
				globals.logger.warning("HTTP Code {} while trying to add user '{}' and checking its validity.".format(e.code, user_name))
				return False, "An error occured when we checked this username on MyAnimeList, maybe the website is down?"
	elif service == utils.Service.ANILIST:
		is_user_valid = anilist.check_username_validity(user_name)
		if is_user_valid == False:
			globals.logger.warning("No results returned while trying to add user '{}' and checking its validity.".format(user_name))
			return False, "User **{}** doesn't exist on AniList!".format(user_name)
	return True, None


async def add_user_cmd(words, message):
	''' Processes the command "add" and add a user to fetch the data for '''

	# Check if command is valid
	if len(words) != 4:
		if (len(words) < 4):
			return await message.channel.send("Usage: {} add **{}**/**{}** **username**".format(globals.prefix, globals.SERVICE_MAL, globals.SERVICE_ANILIST))
		return await message.channel.send("Too many arguments! You have to specify only one username.")

	try:
		service = utils.Service.from_str(words[2])
	except NotImplementedError:
		return await message.channel.send('Incorrect service. Use **"{}"** or **"{}"** for example'.format(globals.SERVICE_MAL, globals.SERVICE_ANILIST))
	user = words[3]
	server_id = str(message.guild.id)

	if(len(user) > 14):
		return await message.channel.send("Username too long!")

	try:
		# Check user validity
		is_valid, error_string = check_user_name_validity(user, service)
		if is_valid == False:
			return await message.channel.send(error_string)

		# Get user's servers
		user_servers = utils.get_user_servers(user, service)
		# User not present in database
		if user_servers is None: 
			utils.insert_user_into_db(user, service, server_id)
			return await message.channel.send("**{}** added to the database for the server **{}**.".format(user, str(message.guild)))
		else: # User present in database

			is_server_present = server_id in user_servers.split(',')
			if is_server_present == True: # The user already has registered this server
				return await message.channel.send("User **{}** is already registered in our database for this server!".format(user))
			else:
				new_servers = '{},{}'.format(user_servers, server_id)
				utils.update_user_servers_db(user, service, new_servers)					
				return await message.channel.send("**{}** added to the database for the server **{}**.".format(user, str(message.guild)))
	except Exception as e:
		globals.logger.warning("Error while adding user '{}' on server '{}': {}".format(user, message.guild, str(e)))
		return await message.channel.send("An unknown error occured while addind this user, the error has been logged.")


async def delete_user_cmd(words, message):
	''' Processes the command "delete" and remove a registered user '''

	# Check if command is valid
	if len(words) != 4:
		if (len(words) < 4):
			return await message.channel.send("Usage: {} delete **{}**/**{}** **username**".format(globals.prefix, globals.SERVICE_MAL, globals.SERVICE_ANILIST))
		return await message.channel.send("Too many arguments! You have to specify only one username.")
	try:
		service = utils.Service.from_str(words[2])
	except NotImplementedError:
		return await message.channel.send('Incorrect service. Use **"{}"** or **"{}"** for example'.format(globals.SERVICE_MAL, globals.SERVICE_ANILIST))
	user = words[3]
	server_id = str(message.guild.id)
	
	user_servers = utils.get_user_servers(user, service)
	# If user is not present in the database
	if user_servers is None:
		return await message.channel.send("The user **" + user + "** is not in our database for this server!")

	# Else if present, update the servers for this user
	srv_string = utils.remove_server_from_servers(server_id, user_servers)
	
	if srv_string is None: # Server not present in the user's servers
		return await message.channel.send("The user **" + user + "** is not in our database for this server!")

	if srv_string == "":
		utils.delete_user_from_db(user, service)
	else:
		utils.update_user_servers_db(user, service, srv_string)

	return await message.channel.send("**" + user + "** deleted from the database for this server.")


@globals.client.event
async def on_message(message):
	if message.author == globals.client.user: return

	words = message.content.split(" ")
	author = str('{0.author.mention}'.format(message))

	# A user is trying to get help
	if words[0] == globals.prefix:
		if len(words) > 1:
			if words[1] == "ping":
				await message.channel.send("pong")
			
			elif words[1] == "here":
				if message.author.guild_permissions.administrator:
					cursor = globals.conn.cursor(buffered=True)
					cursor.execute("SELECT server, channel FROM t_servers WHERE server=%s", [str(message.guild.id)])
					data = cursor.fetchone()
					
					if data is None:
						cursor.execute("INSERT INTO t_servers (server, channel) VALUES (%s,%s)", [str(message.guild.id), str(message.channel.id)])
						globals.conn.commit()
						
						await message.channel.send("Channel **" + str(message.channel) + "** configured for **" + str(message.guild) + "**.")
					else:
						if(data[1] == str(message.channel.id)): await message.channel.send("Channel **" + str(message.channel) + "** already in use for this server.")
						else:
							cursor.execute("UPDATE t_servers SET channel = %s WHERE server = %s", [str(message.channel.id), str(message.guild.id)])
							globals.conn.commit()
							
							await message.channel.send("Channel updated to: **" + str(message.channel) + "**.")
							
					cursor.close()
				else: await message.channel.send("Only server's admins can use this command!")
				
			elif words[1] == "add":
				await add_user_cmd(words, message)
				
			elif words[1] == "delete":
				await delete_user_cmd(words, message)
				
			elif words[1] == "stop":
				if message.author.guild_permissions.administrator:
					if (len(words) == 2):
						cursor = globals.conn.cursor(buffered=True)
						cursor.execute("SELECT server FROM t_servers WHERE server=%s", [str(message.guild.id)])
						data = cursor.fetchone()
					
						if data is None: await globals.client.send_message(message.channel, "The server **" + str(message.guild) + "** is not in our database.")
						else:
							cursor.execute("DELETE FROM t_servers WHERE server = %s", [message.guild.id])
							globals.conn.commit()
							await message.channel.send("Server **" + str(message.guild) + "** deleted from our database.")
						
						cursor.close()
					else: await message.channel.send("Too many arguments! Only type *stop* if you want to stop this bot on **" + message.guild + "**")
				else: await message.channel.send("Only server's admins can use this command!")
				
			elif words[1] == "info":
				await info_cmd(message, words)

			elif words[1] == "about": await message.channel.send(embed=discord.Embed(colour=0x777777, title="MyAnimeBot version " + globals.VERSION + " by Penta & lulu", description="This bot check the MyAnimeList and Anilist profiles for each user specified, and send a message if there is something new.\nMore help with the **" + globals.prefix + " help** command.\n\nCheck the GitHub page: https://github.com/Penta/MyAnimeBot").set_thumbnail(url=globals.iconBot))
			
			elif words[1] == "help": await message.channel.send(globals.HELP)
			
			elif words[1] == "top":
				if len(words) == 2:
					try:
						cursor = globals.conn.cursor(buffered=True)
						cursor.execute("SELECT * FROM v_Top")
						data = cursor.fetchone()
						
						if data is None: await message.channel.send("It seems that there is no statistics... (what happened?!)")
						else:
							topText = "**__Here is the global statistics of this bot:__**\n\n"
							
							while data is not None:
								topText += " - " + str(data[0]) + ": " + str(data[1]) + "\n"
									
								data = cursor.fetchone()
								
							cursor = globals.conn.cursor(buffered=True)
							cursor.execute("SELECT * FROM v_TotalFeeds")
							data = cursor.fetchone()
							
							topText += "\n***Total user entry***: " + str(data[0])
							
							cursor = globals.conn.cursor(buffered=True)
							cursor.execute("SELECT * FROM v_TotalAnimes")
							data = cursor.fetchone()
							
							topText += "\n***Total unique manga/anime***: " + str(data[0])
							
							await message.channel.send(topText)
						
						cursor.close()
					except Exception as e:
						globals.logger.warning("An error occured while displaying the global top: " + str(e))
						await message.channel.send("Unable to reply to your request at the moment...")
				elif len(words) > 2:
					keyword = str(' '.join(words[2:]))
					globals.logger.info("Displaying the global top for the keyword: " + keyword)
					
					try:
						cursor = globals.conn.cursor(buffered=True)
						cursor.callproc('sp_UsersPerKeyword', [str(keyword), '20'])
						for result in cursor.stored_results():
							data = result.fetchone()
							
							if data is None: await message.channel.send("It seems that there is no statistics for the keyword **" + keyword + "**.")
							else:
								topKeyText = "**__Here is the statistics for the keyword " + keyword + ":__**\n\n"
								
								while data is not None:
									topKeyText += " - " + str(data[0]) + ": " + str(data[1]) + "\n"
										
									data = result.fetchone()
									
								await message.channel.send(topKeyText)
							
						cursor.close()
					except Exception as e:
						globals.logger.warning("An error occured while displaying the global top for keyword '" + keyword + "': " + str(e))
						await message.channel.send("Unable to reply to your request at the moment...")
			
			elif words[1] == "group":
				if len(words) > 2:
					if message.author.guild_permissions.administrator:
						group = words[2]
						await message.channel.send("admin OK")
					else: await message.channel.send("Only server's admins can use this command!")
				else:
					await message.channel.send("You have to specify a group!")

			elif words[1] == "fetch-debug":
				await fetch_activities_anilist()

	# If mentioned
	elif globals.client.user in message.mentions:
		await message.channel.send(":heart:")

# Get a random anime name and change the bot's activity
async def change_gameplayed(asyncioloop):
	globals.logger.info("Starting up change_gameplayed")
	
	await globals.client.wait_until_ready()
	await asyncio.sleep(1)

	while not globals.client.is_closed():
		# Get a random anime name from the users' list
		cursor = globals.conn.cursor(buffered=True)
		cursor.execute("SELECT title FROM t_animes ORDER BY RAND() LIMIT 1")
		data = cursor.fetchone()
		anime = utils.truncate_end_show(data[0])
		
		# Try to change the bot's activity
		try:
			if data is not None: await globals.client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=anime))
		except Exception as e:
			globals.logger.warning("An error occured while changing the displayed anime title: " + str(e))
			
		cursor.close()
		# Do it every minute
		await asyncio.sleep(60)

async def update_thumbnail_catalog(asyncioloop):
	globals.logger.info("Starting up update_thumbnail_catalog")
	
	while not globals.client.is_closed():
		await asyncio.sleep(43200)
		
		globals.logger.info("Automatic check of the thumbnail database on going...")
		reload = 0
		
		cursor = globals.conn.cursor(buffered=True)
		cursor.execute("SELECT guid, title, thumbnail FROM t_animes")
		data = cursor.fetchone()

		while data is not None:
			try:
				if (data[2] != "") : urllib.request.urlopen(data[2])
				else: reload = 1
			except urllib.error.HTTPError as e:
				globals.logger.warning("HTTP Error while getting the current thumbnail of '" + str(data[1]) + "': " + str(e))
				reload = 1
			except Exception as e:
				globals.logger.debug("Error while getting the current thumbnail of '" + str(data[1]) + "': " + str(e))
			
			if (reload == 1) :
				try:
					image = myanimelist.get_thumbnail(data[0])
						
					cursor.execute("UPDATE t_animes SET thumbnail = %s WHERE guid = %s", [image, data[0]])
					globals.conn.commit()
						
					globals.logger.info("Updated thumbnail found for \"" + str(data[1]) + "\": %s", image)
				except Exception as e:
					globals.logger.warning("Error while downloading updated thumbnail for '" + str(data[1]) + "': " + str(e))

			await asyncio.sleep(3)
			data = cursor.fetchone()

		cursor.close()

		globals.logger.info("Thumbnail database checked.")
	
# Starting main function	
if __name__ == "__main__":
    try:
        globals.client.run(globals.token)
    except:
        logging.info("Closing all tasks...")
        globals.task_feed.cancel()
        globals.task_feed_anilist.cancel()
        globals.task_thumbnail.cancel()
        globals.task_gameplayed.cancel()

    globals.logger.critical("Script halted.")

	# We close all the ressources
    globals.conn.close()
    globals.log_cursor.close()
    globals.log_conn.close()