myanimebot/myanimebot.py

#!/usr/bin/env python3
# Copyright Penta (c) 2018/2020 - Under BSD License - Based on feed2discord.py by Eric Eisenhart

# Compatible for Python 3.7.X
#
# Dependencies (for CentOS 7):
# yum install python3 mariadb-devel gcc python3-devel
# python3.7 -m pip install --upgrade pip
# pip3.7 install discord.py mysql pytz feedparser python-dateutil asyncio html2text bs4 PyNaCL aiodns cchardet configparser
# pip3.7 install mysql.connector

# Library import
import logging
import os
import sys
import discord
import feedparser
import pytz
import aiohttp
import asyncio
import urllib.request
import mysql.connector as mariadb
import string
import time
import socket

# Custom libraries
sys.path.append('include/')
import utils

from configparser import ConfigParser
from datetime import datetime
from dateutil.parser import parse as parse_datetime
from html2text import HTML2Text
from aiohttp.web_exceptions import HTTPError, HTTPNotModified

if not sys.version_info[:2] >= (3, 7):
	print("ERROR: Requires python 3.7 or newer.")
	exit(1)

class ImproperlyConfigured(Exception): pass

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
HOME_DIR = os.path.expanduser("~")

DEFAULT_CONFIG_PATHS = [
	os.path.join("myanimebot.conf"),
	os.path.join(BASE_DIR, "myanimebot.conf"),
	os.path.join("/etc/malbot/myanimebot.conf"),
	os.path.join(HOME_DIR, "myanimebot.conf")
]

def get_config():
	config = ConfigParser()
	config_paths = []

	for path in DEFAULT_CONFIG_PATHS:
		if os.path.isfile(path):
			config_paths.append(path)
			break
	else: raise ImproperlyConfigured("No configuration file found")
		
	config.read(config_paths)

	return config


# Loading configuration
try:
	config=get_config()
except Exception as e:
	print ("Cannot read configuration: " + str(e))
	exit (1)

CONFIG=config["MYANIMEBOT"]
logLevel=CONFIG.get("logLevel", "INFO")
dbHost=CONFIG.get("dbHost", "127.0.0.1")
dbUser=CONFIG.get("dbUser", "myanimebot")
dbPassword=CONFIG.get("dbPassword")
dbName=CONFIG.get("dbName", "myanimebot")
logPath=CONFIG.get("logPath", "myanimebot.log")
timezone=pytz.timezone(CONFIG.get("timezone", "utc"))
secondMax=CONFIG.getint("secondMax", 7200)
token=CONFIG.get("token")
prefix=CONFIG.get("prefix", "!malbot")
iconMAL=CONFIG.get("iconMAL", "https://cdn.myanimelist.net/img/sp/icon/apple-touch-icon-256.png")
iconBot=CONFIG.get("iconBot", "http://myanimebot.pentou.eu/rsc/bot_avatar.jpg")

# class that send logs to DB
class LogDBHandler(logging.Handler):
	'''
	Customized logging handler that puts logs to the database.
	pymssql required
	'''
	def __init__(self, sql_conn, sql_cursor):
		logging.Handler.__init__(self)
		self.sql_cursor = sql_cursor
		self.sql_conn   = sql_conn

	def emit(self, record):	
		# Clear the log message so it can be put to db via sql (escape quotes)
		self.log_msg = str(record.msg.strip().replace('\'', '\'\''))
		
		# Make the SQL insert
		try:
			self.sql_cursor.execute("INSERT INTO t_logs (host, level, type, log, date, source) VALUES (%s, %s, %s, %s, NOW(), %s)", (str(socket.gethostname()), str(record.levelno), str(record.levelname), self.log_msg, str(record.name)))
			self.sql_conn.commit()
		except Exception as e:
			print ('Error while logging into DB: ' + str(e))

# Log configuration
log_format='%(asctime)-13s : %(name)-15s : %(levelname)-8s : %(message)s'
logging.basicConfig(handlers=[logging.FileHandler(logPath, 'a', 'utf-8')], format=log_format, level=logLevel)

console = logging.StreamHandler()
console.setLevel(logging.INFO)
console.setFormatter(logging.Formatter(log_format))

logger = logging.getLogger("myanimebot")
logger.setLevel(logLevel)

logging.getLogger('').addHandler(console)

# Script version
VERSION = "0.9.6.2"

# The help message
HELP = 	"""**Here's some help for you:**
```
- here :
Type this command on the channel where you want to see the activity of the MAL profiles.

- stop :
Cancel the here command, no message will be displayed.

- add :
Followed by a username, add a MAL user into the database to be displayed on this server.
ex: !malbot add MyUser

- delete :
Followed by a username, remove a user from the database.
ex: !malbot delete MyUser

- group :
Specify a group that can use the add and delete commands.

- info :
Get the users already in the database for this server.

- about :
Get some information about this bot.
```"""

logger.info("Booting MyAnimeBot " + VERSION + "...")
logger.debug("DEBUG log: OK")

# Initialization of the web client
httpclient = aiohttp.ClientSession()

feedparser.PREFERRED_XML_PARSERS.remove("drv_libxml2")

# Initialization of the database
try:
	# Main database connection
	conn = mariadb.connect(host=dbHost, user=dbUser, password=dbPassword, database=dbName, buffered=True)
	
	# We initialize the logs into the DB.
	log_conn   = mariadb.connect(host=dbHost, user=dbUser, password=dbPassword, database=dbName, buffered=True)
	log_cursor = log_conn.cursor()
	logdb = LogDBHandler(log_conn, log_cursor)
	logging.getLogger('').addHandler(logdb)
	
	logger.info("The database logger is running.")
except Exception as e:
	logger.critical("Can't connect to the database: " + str(e))
	
	httpclient.close()
	quit()

# Initialization of the Discord client
client = discord.Client()

# Initialization of the thread handler
loop = asyncio.get_event_loop()

task_feed       = None
task_gameplayed = None
task_thumbnail  = None

# Function used to make the embed message related to the animes status
def build_embed(user, item, channel, pubDate, image):
	try:	
		embed = discord.Embed(colour=0xEED000, url=item.link, description="[" + utils.filter_name(item.title) + "](" + item.link + ")\n```" + item.description + "```", timestamp=pubDate.astimezone(pytz.timezone("utc")))
		embed.set_thumbnail(url=image)
		embed.set_author(name=user + "'s MyAnimeList", url="https://myanimelist.net/profile/" + user, icon_url=iconMAL)
		embed.set_footer(text="MyAnimeBot", icon_url=iconBot)
		
		return embed
	except Exception as e:
		logger.error("Error when generating the message: " + str(e))
		return

# Function used to send the embed
@asyncio.coroutine
def send_embed_wrapper(asyncioloop, channelid, client, embed):
	channel = client.get_channel(int(channelid))
	
	try:
		yield from channel.send(embed=embed)
		logger.info("Message sent in channel: " + channelid)
	except Exception as e:
		logger.debug("Impossible to send a message on '" + channelid + "': " + str(e)) 
		return
	
# Main function that check the RSS feeds from MyAnimeList
@asyncio.coroutine
def background_check_feed(asyncioloop):
	logger.info("Starting up background_check_feed")
	
	# We configure the http header
	http_headers = { "User-Agent": "MyAnimeBot Discord Bot v" + VERSION, }
	
	yield from client.wait_until_ready()
	
	logger.debug("Discord client connected, unlocking background_check_feed...")
	
	while not client.is_closed():
		try:
			db_user = conn.cursor(buffered=True)
			db_user.execute("SELECT mal_user, servers FROM t_users")
			data_user = db_user.fetchone()
		except Exception as e:
			logger.critical("Database unavailable! (" + str(e) + ")")
			quit()

		while data_user is not None:
			user=data_user[0]
			stop_boucle = 0
			feed_type = 1
			
			logger.debug("checking user: " + user)
			
			try:
				while stop_boucle == 0 :
					try:
						if feed_type == 1 :
							http_response = yield from httpclient.request("GET", "https://myanimelist.net/rss.php?type=rm&u=" + user, headers=http_headers)
							media = "manga"
						else : 
							http_response = yield from httpclient.request("GET", "https://myanimelist.net/rss.php?type=rw&u=" + user, headers=http_headers)
							media = "anime"
					except Exception as e:
						logger.error("Error while loading RSS (" + str(feed_type) + ") of '" + user + "': " + str(e))
						break

					http_data = yield from http_response.read()
					feed_data = feedparser.parse(http_data)
					
					http_response.close()

					for item in feed_data.entries:
						pubDateRaw = datetime.strptime(item.published, '%a, %d %b %Y %H:%M:%S %z').astimezone(timezone)
						DateTimezone = pubDateRaw.strftime("%z")[:3] + ':' + pubDateRaw.strftime("%z")[3:]
						pubDate = pubDateRaw.strftime("%Y-%m-%d %H:%M:%S")
						
						cursor = conn.cursor(buffered=True)
						cursor.execute("SELECT published, title, url FROM t_feeds WHERE published=%s AND title=%s AND user=%s", [pubDate, item.title, user])
						data = cursor.fetchone()

						if data is None:
							var = datetime.now(timezone) - pubDateRaw
							
							logger.debug(" - " + item.title + ": " + str(var.total_seconds()))
						
							if var.total_seconds() < secondMax:
								logger.info(user + ": Item '" + item.title + "' not seen, processing...")
								
								if item.description.startswith('-') :
									if feed_type == 1 :	item.description = "Re-Reading " + item.description
									else :				item.description = "Re-Watching " + item.description
								
								cursor.execute("SELECT thumbnail FROM t_animes WHERE guid=%s LIMIT 1", [item.guid])
								data_img = cursor.fetchone()
								
								if data_img is None:
									try:
										image = utils.getThumbnail(item.link)
										
										logger.info("First time seeing this " + media + ", adding thumbnail into database: " + image)
									except Exception as e:
										logger.warning("Error while getting the thumbnail: " + str(e))
										image = ""
										
									cursor.execute("INSERT INTO t_animes (guid, title, thumbnail, found, discoverer, media) VALUES (%s, %s, %s, NOW(), %s, %s)", [item.guid, item.title, image, user, media])
									conn.commit()
								else: image = data_img[0]

								type = item.description.partition(" - ")[0]
								
								cursor.execute("INSERT INTO t_feeds (published, title, url, user, found, type) VALUES (%s, %s, %s, %s, NOW(), %s)", (pubDate, item.title, item.guid, user, type))
								conn.commit()
								
								for server in data_user[1].split(","):
									db_srv = conn.cursor(buffered=True)
									db_srv.execute("SELECT channel FROM t_servers WHERE server = %s", [server])
									data_channel = db_srv.fetchone()
									
									while data_channel is not None:
										for channel in data_channel: yield from send_embed_wrapper(asyncioloop, channel, client, build_embed(user, item, channel, pubDateRaw, image))
										
										data_channel = db_srv.fetchone()
					if feed_type == 1:
						feed_type = 0
						yield from asyncio.sleep(1)
					else:
						stop_boucle = 1
					
			except Exception as e:
				logger.error("Error when parsing RSS for '" + user + "': " + str(e))
			
			yield from asyncio.sleep(1)

			data_user = db_user.fetchone()

@client.event
async def on_ready():
	logger.info("Logged in as " + client.user.name + " (" + str(client.user.id) + ")")

@client.event
async def on_error(event, *args, **kwargs):
    logger.exception("Crap! An unknown Discord error occured...")

@client.event
async def on_message(message):
	if message.author == client.user: return

	words = message.content.split(" ")
	author = str('{0.author.mention}'.format(message))

	# A user is trying to get help
	if words[0] == prefix:
		if len(words) > 1:
			if words[1] == "ping": await message.channel.send("pong")
			
			elif words[1] == "here":
				if message.author.guild_permissions.administrator:
					cursor = conn.cursor(buffered=True)
					cursor.execute("SELECT server, channel FROM t_servers WHERE server=%s", [str(message.guild.id)])
					data = cursor.fetchone()
					
					if data is None:
						cursor.execute("INSERT INTO t_servers (server, channel) VALUES (%s,%s)", [str(message.guild.id), str(message.channel.id)])
						conn.commit()
						
						await message.channel.send("Channel **" + str(message.channel) + "** configured for **" + str(message.guild) + "**.")
					else:
						if(data[1] == str(message.channel.id)): await message.channel.send("Channel **" + str(message.channel) + "** already in use for this server.")
						else:
							cursor.execute("UPDATE t_servers SET channel = %s WHERE server = %s", [str(message.channel.id), str(message.guild.id)])
							conn.commit()
							
							await message.channel.send("Channel updated to: **" + str(message.channel) + "**.")
							
					cursor.close()
				else: await message.channel.send("Only server's admins can use this command!")
				
			elif words[1] == "add":
				if len(words) > 2:
					if (len(words) == 3):
						user = words[2]
						
						if(len(user) < 15):
							try:
								urllib.request.urlopen('https://myanimelist.net/profile/' + user)
								
								cursor = conn.cursor(buffered=True)
								cursor.execute("SELECT servers FROM t_users WHERE LOWER(mal_user)=%s", [user.lower()])
								data = cursor.fetchone()

								if data is None:
									cursor.execute("INSERT INTO t_users (mal_user, servers) VALUES (%s, %s)", [user, str(message.guild.id)])
									conn.commit()
									
									await message.channel.send("**" + user + "** added to the database for the server **" + str(message.guild) + "**.")
								else:
									var = 0
									
									for server in data[0].split(","):
										if (server == str(message.guild.id)): var = 1
									
									if (var == 1):
										await message.channel.send("User **" + user + "** already in our database for this server!")
									else:
										cursor.execute("UPDATE t_users SET servers = %s WHERE LOWER(mal_user) = %s", [data[0] + "," + str(message.guild.id), user.lower()])
										conn.commit()
										
										await message.channel.send("**" + user + "** added to the database for the server **" + str(message.guild) + "**.")
										
								cursor.close()
							except urllib.error.HTTPError as e:
								if (e.code == 404): await message.channel.send("User **" + user + "** doesn't exist on MyAnimeList!")
								else:
									await message.channel.send("An error occured when we checked this username on MyAnimeList, maybe the website is down?")
									logger.warning("HTTP Code " + str(e.code) + " while checking to add for the new user '" + user + "'")
							except Exception as e:
								await message.channel.send("An unknown error occured while addind this user, the error has been logged.")
								logger.warning("Error while adding user '" + user + "' on server '" + message.guild + "': " + str(e))
						else: await message.channel.send("Username too long!")
					else: await message.channel.send("Too many arguments! You have to specify only one username.")
				else: await message.channel.send("You have to specify a **MyAnimeList** username!")
				
			elif words[1] == "delete":
				if len(words) > 2:
					if (len(words) == 3):
						user = words[2]
						
						cursor = conn.cursor(buffered=True)
						cursor.execute("SELECT servers FROM t_users WHERE LOWER(mal_user)=%s", [user.lower()])
						data = cursor.fetchone()
						
						if data is not None:
							srv_string = ""
							present = 0
							
							for server in data[0].split(","):
								if server != str(message.guild.id):
									if srv_string == "": srv_string = server
									else: srv_string += "," + server
								else: present = 1
							
							if present == 1:
								if srv_string == "": cursor.execute("DELETE FROM t_users WHERE LOWER(mal_user) = %s", [user.lower()])
								else: cursor.execute("UPDATE t_users SET servers = %s WHERE LOWER(mal_user) = %s", [srv_string, user.lower()])
								conn.commit()
								
								await message.channel.send("**" + user + "** deleted from the database for this server.")
							else: await message.channel.send("The user **" + user + "** is not in our database for this server!")
						else: await message.channel.send("The user **" + user + "** is not in our database for this server!")
			
						cursor.close()
					else: await message.channel.send("Too many arguments! You have to specify only one username.")
				else: await message.channel.send("You have to specify a **MyAnimeList** username!")
				
			elif words[1] == "stop":
				if message.author.guild_permissions.administrator:
					if (len(words) == 2):
						cursor = conn.cursor(buffered=True)
						cursor.execute("SELECT server FROM t_servers WHERE server=%s", [str(message.guild.id)])
						data = cursor.fetchone()
					
						if data is None: await client.send_message(message.channel, "The server **" + str(message.guild) + "** is not in our database.")
						else:
							cursor.execute("DELETE FROM t_servers WHERE server = %s", [message.guild.id])
							conn.commit()
							await message.channel.send("Server **" + str(message.guild) + "** deleted from our database.")
						
						cursor.close()
					else: await message.channel.send("Too many arguments! Only type *stop* if you want to stop this bot on **" + message.guild + "**")
				else: await message.channel.send("Only server's admins can use this command!")
				
			elif words[1] == "info":
				cursor = conn.cursor(buffered=True)
				cursor.execute("SELECT server FROM t_servers WHERE server=%s", [str(message.guild.id)])
				data = cursor.fetchone()
				
				if data is None: await message.channel.send("The server **" + str(message.guild) + "** is not in our database.")
				else:
					user = ""
					cursor = conn.cursor(buffered=True)
					cursor.execute("SELECT mal_user, servers FROM t_users")
					data = cursor.fetchone()
					
					cursor_channel = conn.cursor(buffered=True)
					cursor_channel.execute("SELECT channel FROM t_servers WHERE server=%s", [str(message.guild.id)])
					data_channel = cursor_channel.fetchone()
					
					if data_channel is None: await message.channel.send("No channel assigned for this bot in this server.")
					else:
						while data is not None:
							if (str(message.guild.id) in data[1].split(",")):
								if (user == ""): user = data[0]
								else: user += ", " + data[0]
						
							data = cursor.fetchone()
						
						if (user == ""): await message.channel.send("No user in this server.")
						else: await message.channel.send("Here's the user(s) in the **" + str(message.guild) + "**'s server:\n```" + user + "```\nAssigned channel: **" + str(client.get_channel(int(data_channel[0]))) + "**")

					cursor.close()
					cursor_channel.close()
			elif words[1] == "about": await message.channel.send(embed=discord.Embed(colour=0x777777, title="MyAnimeBot version " + VERSION + " by Penta", description="This bot check the MyAnimeList's RSS for each user specified, and send a message if there is something new.\nMore help with the **!malbot help** command.\n\nAdd me on steam: http://steamcommunity.com/id/Penta_Pingouin").set_thumbnail(url="https://cdn.discordapp.com/avatars/415474467033317376/2d847944aab2104923c18863a41647da.jpg?size=64"))
			
			elif words[1] == "help": await message.channel.send(HELP)
			
			elif words[1] == "top":
				if len(words) == 2:
					try:
						cursor = conn.cursor(buffered=True)
						cursor.execute("SELECT * FROM v_Top")
						data = cursor.fetchone()
						
						if data is None: await message.channel.send("It seems that there is no statistics... (what happened?!)")
						else:
							topText = "**__Here is the global statistics of this bot:__**\n\n"
							
							while data is not None:
								topText += " - " + str(data[0]) + ": " + str(data[1]) + "\n"
									
								data = cursor.fetchone()
								
							cursor = conn.cursor(buffered=True)
							cursor.execute("SELECT * FROM v_TotalFeeds")
							data = cursor.fetchone()
							
							topText += "\n***Total user entry***: " + str(data[0])
							
							cursor = conn.cursor(buffered=True)
							cursor.execute("SELECT * FROM v_TotalAnimes")
							data = cursor.fetchone()
							
							topText += "\n***Total unique manga/anime***: " + str(data[0])
							
							await message.channel.send(topText)
						
						cursor.close()
					except Exception as e:
						logger.warning("An error occured while displaying the global top: " + str(e))
						await message.channel.send("Unable to reply to your request at the moment...")
				elif len(words) > 2:
					keyword = str(' '.join(words[2:]))
					logger.info("Displaying the global top for the keyword: " + keyword)
					
					try:
						cursor = conn.cursor(buffered=True)
						cursor.callproc('sp_UsersPerKeyword', [str(keyword), '20'])
						for result in cursor.stored_results():
							data = result.fetchone()
							
							if data is None: await message.channel.send("It seems that there is no statistics for the keyword **" + keyword + "**.")
							else:
								topKeyText = "**__Here is the statistics for the keyword " + keyword + ":__**\n\n"
								
								while data is not None:
									topKeyText += " - " + str(data[0]) + ": " + str(data[1]) + "\n"
										
									data = result.fetchone()
									
								await message.channel.send(topKeyText)
							
						cursor.close()
					except Exception as e:
						logger.warning("An error occured while displaying the global top for keyword '" + keyword + "': " + str(e))
						await message.channel.send("Unable to reply to your request at the moment...")
			
			elif words[1] == "group":
				if len(words) > 2:
					if message.author.guild_permissions.administrator:
						group = words[2]
						await message.channel.send("admin OK")
					else: await message.channel.send("Only server's admins can use this command!")
				else:
					await message.channel.send("You have to specify a group!")
		
	# If mentioned
	elif client.user in message.mentions:
		await message.channel.send(":heart:")

# Get a random anime name and change the bot's activity
@asyncio.coroutine	
def change_gameplayed(asyncioloop):
	logger.info("Starting up change_gameplayed")
	
	yield from client.wait_until_ready()
	yield from asyncio.sleep(1)

	while not client.is_closed():
		# Get a random anime name from the users' list
		cursor = conn.cursor(buffered=True)
		cursor.execute("SELECT title FROM t_animes ORDER BY RAND() LIMIT 1")
		data = cursor.fetchone()
		anime = utils.truncate_end_show(data[0])
		
		# Try to change the bot's activity
		try:
			if data is not None: yield from client.change_presence(activity=discord.Activity(type=discord.ActivityType.watching, name=anime))
		except Exception as e:
			logger.warning("An error occured while changing the displayed anime title: " + str(e))
			
		cursor.close()
		# Do it every minute
		yield from asyncio.sleep(60)

@asyncio.coroutine	
def update_thumbnail_catalog(asyncioloop):
	logger.info("Starting up update_thumbnail_catalog")
	
	while not client.is_closed():
		yield from asyncio.sleep(43200)
		
		logger.info("Automatic check of the thumbnail database on going...")
		reload = 0
		
		cursor = conn.cursor(buffered=True)
		cursor.execute("SELECT guid, title, thumbnail FROM t_animes")
		data = cursor.fetchone()

		while data is not None:
			try:
				if (data[2] != "") : urllib.request.urlopen(data[2])
				else: reload = 1
			except urllib.error.HTTPError as e:
				logger.warning("HTTP Error while getting the current thumbnail of '" + str(data[1]) + "': " + str(e))
				reload = 1
			except Exception as e:
				logger.debug("Error while getting the current thumbnail of '" + str(data[1]) + "': " + str(e))
			
			if (reload == 1) :
				try:
					image = utils.getThumbnail(data[0])
						
					cursor.execute("UPDATE t_animes SET thumbnail = %s WHERE guid = %s", [image, data[0]])
					conn.commit()
						
					logger.info("Updated thumbnail found for \"" + str(data[1]) + "\": %s", image)
				except Exception as e:
					logger.warning("Error while downloading updated thumbnail for '" + str(data[1]) + "': " + str(e))

			yield from asyncio.sleep(3)
			data = cursor.fetchone()

		cursor.close()

		logger.info("Thumbnail database checked.")

def main():
	logger.info("Starting all tasks...")

	try:
		task_feed = loop.create_task(background_check_feed(loop))
		task_thumbnail = loop.create_task(update_thumbnail_catalog(loop))
		task_gameplayed = loop.create_task(change_gameplayed(loop))
	
		client.run(token)
	except:
		logging.info("Closing all tasks...")
		
		task_feed.cancel()
		task_thumbnail.cancel()
		task_gameplayed.cancel()
		
		loop.run_until_complete(client.close())
	finally:
		loop.close()
	
# Starting main function	
if __name__ == "__main__":
	main()

	logger.critical("Script halted.")
	
	# We close all the ressources
	conn.close()
	log_cursor.close()
	log_conn.close()
	httpclient.close()
	loop.stop()
	loop.close()