Python script that scrapes rom sites (right now only vimm.net) to find and download ROMs
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 

94 lines
3.0 KiB

import os
from random import random
import requests
from requests_html import HTMLSession
from time import sleep
import re
from pprint import pprint
from Models.Website import Website
from Models.Console import Console
from Models.Game import Game
from dbHandler import dbConnection
site = Website(website_id=1)
def searchGames(gamename, consolename="", fuzzyMatch=True):
db = dbConnection()
games = db.searchByName("Games", gamename, fuzzyMatch)
return games
def indexGame(gameId, website):
print(website.url)
url = website.url + str(gameId)
gamefound = False
try:
session = HTMLSession()
response = session.get(url)
if len(response.html.find('h2')) > 0:
# This part gets the game info
console = Console(name=response.html.find('h2')[0].text.split('\n')[0])
game = Game(name=response.html.find('h2')[0].text.split('\n')[1])
game.addDownloadUrl(site, url, console.console_id)
game.addConsole(console)
print("GameID: " + str(game.game_id))
print("Game Name: " + game.name)
print("Game Console: " + game.console)
print("Download URL: " + game.downloadURL)
print("ConsoleID: " + str(console.console_id))
gamefound = True
else:
print("There is no game with this ID")
except requests.exceptions.RequestException as e:
print(e)
return gamefound
# gameID = 25200
# consecutiveFailedURLs = 0
# while consecutiveFailedURLs < 20:
# if not indexGame(gameID, website=Website(website_id=1)):
# consecutiveFailedURLs += 1
# else:
# consecutiveFailedURLs = 0
# gameID += 1
# sleep(random())
# if gameID % 200 == 0:
# sleep(random()*100)
#
# print("Last ID: " + gameID)
# game = Game(game_id=483)
# print(game.name)
# game.download(site, Console(console_id=1))
print("Enter a game name (full or partial) that you'd like to search for:")
searchterm = input()
games = searchGames(searchterm, True)
gamedownloadarr = []
for game in games:
gameobj = Game(game[0])
# print("\n\n"+gameobj.name + "\n__________________________\n")
for row in gameobj.downloadURLArr:
gamedownloadarr.append({"game": gameobj.name, 'game_id': gameobj.game_id, "console_id": row['console_id'], "url": row['url']})
i = 1
for game in gamedownloadarr:
print(str(i) + ") " + game['game'] + " - " + Console(console_id=game['console_id']).name + ": " + game['url'])
i += 1
print("What would you like to download?")
print("You can download multiple by typing multiple numbers with spaces between, ex: \"1 2 3\"...")
downloadSelRaw = input()
downloadlist = downloadSelRaw.split()
for selection in downloadlist:
selection = int(selection)
gamedownload = Game(game_id=gamedownloadarr[selection-1]['game_id'])
gamedownload.download(site, Console(gamedownloadarr[selection-1]['console_id']))
print("Waiting so server doesn't block our requests.....")
sleep(10)