forked from olmstea1/python-romscraper
Initial Commit
This commit is contained in:
aolmstead
93
Models/Console.py
Normal file
93
Models/Console.py
Normal file
@ -0,0 +1,93 @@
|
||||
import sqlite3
|
||||
from dbHandler import dbConnection
|
||||
|
||||
|
||||
class Console:
|
||||
console_id = 0
|
||||
name = ""
|
||||
description = ""
|
||||
thumbnail_uri = ""
|
||||
rating = 0
|
||||
|
||||
def __init__(self, console_id=None, name=None):
|
||||
if console_id is None and name is not None:
|
||||
self.name = name
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
# print("Connected")
|
||||
query = "SELECT * FROM Consoles WHERE name=?"
|
||||
cursor.execute(query, [name, ])
|
||||
records = cursor.fetchall()
|
||||
if len(records) < 1:
|
||||
# print("Console is not in table")
|
||||
query = "INSERT INTO Consoles (name) VALUES (?)"
|
||||
cursor.execute(query, [name, ])
|
||||
sqliteConnection.commit()
|
||||
query = "SELECT seq FROM sqlite_sequence WHERE name=?"
|
||||
cursor.execute(query, ["Consoles"])
|
||||
records = cursor.fetchall()
|
||||
for row in records:
|
||||
self.console_id = row[0]
|
||||
cursor.close()
|
||||
else:
|
||||
for row in records:
|
||||
self.console_id = row[0]
|
||||
self.populateConsoleAttrs()
|
||||
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
|
||||
finally:
|
||||
if (sqliteConnection):
|
||||
sqliteConnection.close()
|
||||
print("The SQLite connection is closed")
|
||||
|
||||
elif name is None and console_id is not None:
|
||||
self.console_id = console_id
|
||||
db = dbConnection()
|
||||
query = "SELECT * FROM Consoles WHERE console_id=?"
|
||||
values = [console_id, ]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
if records.__sizeof__() > 0:
|
||||
self.populateConsoleAttrs()
|
||||
db.closeDB()
|
||||
|
||||
else:
|
||||
print("No data given, initializing blank Console object")
|
||||
|
||||
def updateConsoleAttrs(self, console_id=None):
|
||||
if console_id is None:
|
||||
# print("Console ID is REQUIRED!")
|
||||
return
|
||||
else:
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
# print("Connected")
|
||||
query = "SELECT * FROM Consoles WHERE console_id=?"
|
||||
cursor.execute(query, [console_id, ])
|
||||
records = cursor.fetchall()
|
||||
if len(records) > 0:
|
||||
# print("Console is not in table")
|
||||
query = "UPDATE Consoles SET name=?, description=?, rating=?, thumbnail_uri=? WHERE console_id=?"
|
||||
cursor.execute(query,
|
||||
[self.name, self.description, self.rating, self.thumbnail_uri, self.console_id])
|
||||
sqliteConnection.commit()
|
||||
cursor.close()
|
||||
else:
|
||||
print("Console with this ID does not exist!")
|
||||
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
|
||||
def populateConsoleAttrs(self):
|
||||
db = dbConnection()
|
||||
query = "SELECT * FROM Consoles WHERE console_id=?"
|
||||
values = [self.console_id]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
for row in records:
|
||||
self.name = row[1]
|
||||
self.rating = row[4]
|
||||
self.description = row[2]
|
||||
return
|
||||
0
Models/Emulator.py
Normal file
0
Models/Emulator.py
Normal file
225
Models/Game.py
Normal file
225
Models/Game.py
Normal file
@ -0,0 +1,225 @@
|
||||
import sqlite3
|
||||
from dbHandler import dbConnection
|
||||
import requests
|
||||
from requests_html import HTMLSession
|
||||
import re
|
||||
from pprint import pprint
|
||||
import os
|
||||
from time import sleep
|
||||
|
||||
class Game:
|
||||
game_id = 0
|
||||
name = ""
|
||||
description = ""
|
||||
thumbnail_uri = ""
|
||||
rating = 0
|
||||
downloadURL = ""
|
||||
downloadURLArr = []
|
||||
console = ""
|
||||
|
||||
def __init__(self, game_id=None, name=None):
|
||||
self.game_id = 0
|
||||
self.name = ""
|
||||
self.description = ""
|
||||
self.thumbnail_uri = ""
|
||||
self.rating = 0
|
||||
self.downloadURL = ""
|
||||
self.downloadURLArr = []
|
||||
self.console = ""
|
||||
|
||||
|
||||
if game_id is None and name is not None:
|
||||
self.name = name
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
print("Connected")
|
||||
query = "SELECT * FROM Games WHERE name=?"
|
||||
cursor.execute(query, [name, ])
|
||||
records = cursor.fetchall()
|
||||
if len(records) < 1:
|
||||
print("Game is not in table")
|
||||
query = "INSERT INTO Games (name) VALUES (?)"
|
||||
cursor.execute(query, [name, ])
|
||||
sqliteConnection.commit()
|
||||
query = "SELECT seq FROM sqlite_sequence WHERE name=?"
|
||||
cursor.execute(query, ["Games"])
|
||||
records = cursor.fetchall()
|
||||
for row in records:
|
||||
self.game_id = row[0]
|
||||
cursor.close()
|
||||
|
||||
else:
|
||||
for row in records:
|
||||
self.game_id = row[0]
|
||||
self.populateGameAttrs()
|
||||
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
|
||||
finally:
|
||||
if (sqliteConnection):
|
||||
sqliteConnection.close()
|
||||
print("The SQLite connection is closed")
|
||||
|
||||
elif name is None and game_id is not None:
|
||||
self.game_id = game_id
|
||||
db = dbConnection()
|
||||
query = "SELECT * FROM Games WHERE game_id=?"
|
||||
values = [game_id, ]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
if records.__sizeof__() > 0:
|
||||
self.populateGameAttrs()
|
||||
db.closeDB()
|
||||
|
||||
else:
|
||||
print("No data given, initializing blank game object")
|
||||
|
||||
def populateGameAttrs(self):
|
||||
db = dbConnection()
|
||||
query = "SELECT * FROM Games WHERE game_id=?"
|
||||
values = [self.game_id]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
for row in records:
|
||||
self.name = row[2]
|
||||
self.rating = row[1]
|
||||
self.description = row[3]
|
||||
|
||||
query = "SELECT * FROM Game_On_Website WHERE game_id=?"
|
||||
#print(self.game_id)
|
||||
values = [self.game_id]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
for row in records:
|
||||
self.downloadURLArr.append({'console_id': row[4], 'website_id': row[2], 'url': row[3]})
|
||||
self.downloadURL = row[3]
|
||||
|
||||
query = "SELECT * FROM Game_On_Console WHERE game_id=?"
|
||||
values = [self.game_id]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
for row in records:
|
||||
self.console = row[2]
|
||||
|
||||
return
|
||||
|
||||
def updateGameAttrs(self, game_id=None):
|
||||
if game_id is None:
|
||||
print("Game ID is REQUIRED!")
|
||||
return
|
||||
else:
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
#print("Connected")
|
||||
query = "SELECT * FROM Games WHERE game_id=?"
|
||||
cursor.execute(query, [game_id, ])
|
||||
records = cursor.fetchall()
|
||||
if len(records) > 0:
|
||||
query = "UPDATE Games SET name=?, description=?, rating=?, thumbnail_uri=? WHERE game_id=?"
|
||||
cursor.execute(query, [self.name, self.description, self.rating, self.thumbnail_uri, self.game_id])
|
||||
sqliteConnection.commit()
|
||||
|
||||
else:
|
||||
print("Game with this ID does not exist!")
|
||||
cursor.close()
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
return
|
||||
|
||||
def addDownloadUrl(self, website, url, consoleId):
|
||||
self.downloadURL = url
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
#print("Connected")
|
||||
query = "SELECT * FROM Game_On_Website WHERE game_id=? AND url=?"
|
||||
cursor.execute(query, [self.game_id, url])
|
||||
records = cursor.fetchall()
|
||||
if len(records) > 0:
|
||||
print("Game is already catalogued!")
|
||||
else:
|
||||
query = "INSERT INTO Game_On_Website (game_id, website_id, url, console_id) VALUES (?,?,?, ?)"
|
||||
cursor.execute(query, [self.game_id, website.website_id, url, consoleId])
|
||||
sqliteConnection.commit()
|
||||
cursor.close()
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
|
||||
return
|
||||
|
||||
def addConsole(self, console):
|
||||
self.console = console.name
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
#print("Connected")
|
||||
query = "SELECT * FROM Game_On_Console WHERE game_id=? AND console_id=?"
|
||||
cursor.execute(query, [self.game_id, console.console_id])
|
||||
records = cursor.fetchall()
|
||||
if len(records) > 0:
|
||||
print("Game is already catalogued!")
|
||||
else:
|
||||
query = "INSERT INTO Game_On_Console (game_id, console_id) VALUES (?,?)"
|
||||
cursor.execute(query, [self.game_id, console.console_id])
|
||||
sqliteConnection.commit()
|
||||
cursor.close()
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
|
||||
return
|
||||
|
||||
def download(self, website, console):
|
||||
session = HTMLSession()
|
||||
for loc in self.downloadURLArr:
|
||||
if loc['website_id'] == website.website_id and loc['console_id'] == console.console_id:
|
||||
self.downloadURL = loc['url']
|
||||
|
||||
response = session.get(self.downloadURL)
|
||||
|
||||
|
||||
|
||||
inputs = response.html.find('input')
|
||||
for inputField in inputs:
|
||||
if inputField.html.find('mediaId') != -1:
|
||||
mediaID = re.findall('value=\"(.*?)\"', inputField.html)[0]
|
||||
|
||||
headers = {
|
||||
"User-Agent": 'Mozilla/5.0(X11; Linux x86_64; rv: 83.0) '
|
||||
'Gecko/20100101'
|
||||
'Firefox / 83.0',
|
||||
"Accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
"Accept-Language": 'en-US,en;q=0.5',
|
||||
"Accept-Encoding": 'gzip, deflate, br',
|
||||
"Connection": 'keep-alive',
|
||||
"Referer": self.downloadURL,
|
||||
"Cookie": '__cfduid = d186e9398ba52f9caf7e9a1fdae8b51511606415231',
|
||||
"Upgrade-Insecure-Requests": '1',
|
||||
}
|
||||
serverNum = 4
|
||||
fileDownloaded = False
|
||||
while not fileDownloaded:
|
||||
try:
|
||||
r = requests.get("https://download" + str(serverNum) + ".vimm.net/download/?mediaId=" + str(mediaID),
|
||||
headers=headers)
|
||||
|
||||
except requests.exceptions.RequestException as e:
|
||||
serverNum -= 1
|
||||
print(e)
|
||||
|
||||
else:
|
||||
if r.status_code == 200:
|
||||
pprint("saving " + self.name)
|
||||
h = r.headers['content-disposition']
|
||||
saveLocation = os.path.expanduser('~/Downloads/RomScraper/') + console.name + "/" + self.name[0] + "/" + re.findall("filename=\"(.+)\"", h)[0]
|
||||
os.makedirs(os.path.dirname(saveLocation), exist_ok=True)
|
||||
with open(saveLocation, "w") as f:
|
||||
f.write("FOOBAR")
|
||||
fileDownloaded = True
|
||||
else:
|
||||
pprint("File not found on server" + str(serverNum) + ", trying next server")
|
||||
serverNum -= 1
|
||||
sleep(2)
|
||||
|
||||
if serverNum == 0:
|
||||
pprint("Game was unable to be downloaded!")
|
||||
fileDownloaded = True
|
||||
return
|
||||
77
Models/Website.py
Normal file
77
Models/Website.py
Normal file
@ -0,0 +1,77 @@
|
||||
import sqlite3
|
||||
from dbHandler import dbConnection
|
||||
|
||||
|
||||
class Website:
|
||||
website_id = 0
|
||||
name = ""
|
||||
url = ""
|
||||
description = ""
|
||||
thumbnail_uri = ""
|
||||
rating = 0
|
||||
|
||||
def __init__(self, website_id=None, name=None):
|
||||
if website_id is None and name is not None:
|
||||
self.name = name
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
print("Connected")
|
||||
query = "SELECT * FROM Websites WHERE name=?"
|
||||
cursor.execute(query, [name, ])
|
||||
records = cursor.fetchall()
|
||||
if len(records) < 1:
|
||||
print("Website is not in table")
|
||||
query = "INSERT INTO Websites (name) VALUES (?)"
|
||||
cursor.execute(query, [name, ])
|
||||
sqliteConnection.commit()
|
||||
cursor.close()
|
||||
else:
|
||||
for row in records:
|
||||
self.website_id = row[0]
|
||||
self.url = row[2]
|
||||
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
|
||||
finally:
|
||||
if (sqliteConnection):
|
||||
sqliteConnection.close()
|
||||
print("The SQLite connection is closed")
|
||||
|
||||
elif name is None and website_id is not None:
|
||||
db = dbConnection()
|
||||
query = "SELECT * FROM Websites WHERE website_id=?"
|
||||
values = [website_id, ]
|
||||
records = db.queryDB(query, values, returnsRecords=True)
|
||||
for row in records:
|
||||
self.website_id = row[0]
|
||||
self.url = row[2]
|
||||
db.closeDB()
|
||||
|
||||
else:
|
||||
print("No data given, initializing blank Website object")
|
||||
|
||||
def updateWebsiteAttrs(self, website_id=None):
|
||||
if website_id is None:
|
||||
print("Website ID is REQUIRED!")
|
||||
return
|
||||
else:
|
||||
try:
|
||||
sqliteConnection = sqlite3.connect('Romscraper.db')
|
||||
cursor = sqliteConnection.cursor()
|
||||
print("Connected")
|
||||
query = "SELECT * FROM Websites WHERE website_id=?"
|
||||
cursor.execute(query, [website_id, ])
|
||||
records = cursor.fetchall()
|
||||
if len(records) > 0:
|
||||
print("Website is not in table")
|
||||
query = "UPDATE Websites SET name=?, url=?, description=?, rating=?, thumbnail_uri=? WHERE website_id=? "
|
||||
cursor.execute(query, [self.name, self.url, self.description, self.rating, self.thumbnail_uri, self.website_id])
|
||||
sqliteConnection.commit()
|
||||
cursor.close()
|
||||
else:
|
||||
print("Website with this ID does not exist!")
|
||||
|
||||
except sqlite3.Error as error:
|
||||
print("Failed to fetch data from database: ", error)
|
||||
Reference in New Issue
Block a user