forked from olmstea1/python-romscraper
aolmstead
3 years ago
commit
5faaff8d61
19 changed files with 1062 additions and 0 deletions
@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
|
||||
# Default ignored files |
||||
/shelf/ |
||||
/workspace.xml |
||||
# Datasource local storage ignored files |
||||
/dataSources/ |
||||
/dataSources.local.xml |
||||
# Editor-based HTTP Client requests |
||||
/httpRequests/ |
||||
@ -0,0 +1,17 @@
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project version="4"> |
||||
<component name="DataSourceManagerImpl" format="xml" multifile-model="true"> |
||||
<data-source source="LOCAL" name="Romscraper" uuid="8bc82c4d-3e9a-46f9-ac5f-791bcb2e6de4"> |
||||
<driver-ref>sqlite.xerial</driver-ref> |
||||
<synchronize>true</synchronize> |
||||
<jdbc-driver>org.sqlite.JDBC</jdbc-driver> |
||||
<jdbc-url>jdbc:sqlite:$PROJECT_DIR$/Romscraper.db</jdbc-url> |
||||
<working-dir>$ProjectFileDir$</working-dir> |
||||
<libraries> |
||||
<library> |
||||
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.34.0/sqlite-jdbc-3.34.0.jar</url> |
||||
</library> |
||||
</libraries> |
||||
</data-source> |
||||
</component> |
||||
</project> |
||||
@ -0,0 +1,6 @@
@@ -0,0 +1,6 @@
|
||||
<component name="InspectionProjectProfileManager"> |
||||
<settings> |
||||
<option name="USE_PROJECT_PROFILE" value="false" /> |
||||
<version value="1.0" /> |
||||
</settings> |
||||
</component> |
||||
@ -0,0 +1,4 @@
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project version="4"> |
||||
<component name="ProjectRootManager" version="2" project-jdk-name="Pipenv (romscraper)" project-jdk-type="Python SDK" /> |
||||
</project> |
||||
@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project version="4"> |
||||
<component name="ProjectModuleManager"> |
||||
<modules> |
||||
<module fileurl="file://$PROJECT_DIR$/.idea/pythonProject2.iml" filepath="$PROJECT_DIR$/.idea/pythonProject2.iml" /> |
||||
</modules> |
||||
</component> |
||||
</project> |
||||
@ -0,0 +1,8 @@
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<module type="PYTHON_MODULE" version="4"> |
||||
<component name="NewModuleRootManager"> |
||||
<content url="file://$MODULE_DIR$" /> |
||||
<orderEntry type="inheritedJdk" /> |
||||
<orderEntry type="sourceFolder" forTests="false" /> |
||||
</component> |
||||
</module> |
||||
@ -0,0 +1,7 @@
@@ -0,0 +1,7 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project version="4"> |
||||
<component name="SqlDialectMappings"> |
||||
<file url="file://$PROJECT_DIR$/Models/Console.py" dialect="GenericSQL" /> |
||||
<file url="PROJECT" dialect="SQLite" /> |
||||
</component> |
||||
</project> |
||||
@ -0,0 +1,6 @@
@@ -0,0 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?> |
||||
<project version="4"> |
||||
<component name="VcsDirectoryMappings"> |
||||
<mapping directory="$PROJECT_DIR$" vcs="Git" /> |
||||
</component> |
||||
</project> |
||||
@ -0,0 +1,93 @@
@@ -0,0 +1,93 @@
|
||||
import sqlite3 |
||||
from dbHandler import dbConnection |
||||
|
||||
|
||||
class Console: |
||||
console_id = 0 |
||||
name = "" |
||||
description = "" |
||||
thumbnail_uri = "" |
||||
rating = 0 |
||||
|
||||
def __init__(self, console_id=None, name=None): |
||||
if console_id is None and name is not None: |
||||
self.name = name |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
# print("Connected") |
||||
query = "SELECT * FROM Consoles WHERE name=?" |
||||
cursor.execute(query, [name, ]) |
||||
records = cursor.fetchall() |
||||
if len(records) < 1: |
||||
# print("Console is not in table") |
||||
query = "INSERT INTO Consoles (name) VALUES (?)" |
||||
cursor.execute(query, [name, ]) |
||||
sqliteConnection.commit() |
||||
query = "SELECT seq FROM sqlite_sequence WHERE name=?" |
||||
cursor.execute(query, ["Consoles"]) |
||||
records = cursor.fetchall() |
||||
for row in records: |
||||
self.console_id = row[0] |
||||
cursor.close() |
||||
else: |
||||
for row in records: |
||||
self.console_id = row[0] |
||||
self.populateConsoleAttrs() |
||||
|
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
finally: |
||||
if (sqliteConnection): |
||||
sqliteConnection.close() |
||||
print("The SQLite connection is closed") |
||||
|
||||
elif name is None and console_id is not None: |
||||
self.console_id = console_id |
||||
db = dbConnection() |
||||
query = "SELECT * FROM Consoles WHERE console_id=?" |
||||
values = [console_id, ] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
if records.__sizeof__() > 0: |
||||
self.populateConsoleAttrs() |
||||
db.closeDB() |
||||
|
||||
else: |
||||
print("No data given, initializing blank Console object") |
||||
|
||||
def updateConsoleAttrs(self, console_id=None): |
||||
if console_id is None: |
||||
# print("Console ID is REQUIRED!") |
||||
return |
||||
else: |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
# print("Connected") |
||||
query = "SELECT * FROM Consoles WHERE console_id=?" |
||||
cursor.execute(query, [console_id, ]) |
||||
records = cursor.fetchall() |
||||
if len(records) > 0: |
||||
# print("Console is not in table") |
||||
query = "UPDATE Consoles SET name=?, description=?, rating=?, thumbnail_uri=? WHERE console_id=?" |
||||
cursor.execute(query, |
||||
[self.name, self.description, self.rating, self.thumbnail_uri, self.console_id]) |
||||
sqliteConnection.commit() |
||||
cursor.close() |
||||
else: |
||||
print("Console with this ID does not exist!") |
||||
|
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
def populateConsoleAttrs(self): |
||||
db = dbConnection() |
||||
query = "SELECT * FROM Consoles WHERE console_id=?" |
||||
values = [self.console_id] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
for row in records: |
||||
self.name = row[1] |
||||
self.rating = row[4] |
||||
self.description = row[2] |
||||
return |
||||
@ -0,0 +1,225 @@
@@ -0,0 +1,225 @@
|
||||
import sqlite3 |
||||
from dbHandler import dbConnection |
||||
import requests |
||||
from requests_html import HTMLSession |
||||
import re |
||||
from pprint import pprint |
||||
import os |
||||
from time import sleep |
||||
|
||||
class Game: |
||||
game_id = 0 |
||||
name = "" |
||||
description = "" |
||||
thumbnail_uri = "" |
||||
rating = 0 |
||||
downloadURL = "" |
||||
downloadURLArr = [] |
||||
console = "" |
||||
|
||||
def __init__(self, game_id=None, name=None): |
||||
self.game_id = 0 |
||||
self.name = "" |
||||
self.description = "" |
||||
self.thumbnail_uri = "" |
||||
self.rating = 0 |
||||
self.downloadURL = "" |
||||
self.downloadURLArr = [] |
||||
self.console = "" |
||||
|
||||
|
||||
if game_id is None and name is not None: |
||||
self.name = name |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
print("Connected") |
||||
query = "SELECT * FROM Games WHERE name=?" |
||||
cursor.execute(query, [name, ]) |
||||
records = cursor.fetchall() |
||||
if len(records) < 1: |
||||
print("Game is not in table") |
||||
query = "INSERT INTO Games (name) VALUES (?)" |
||||
cursor.execute(query, [name, ]) |
||||
sqliteConnection.commit() |
||||
query = "SELECT seq FROM sqlite_sequence WHERE name=?" |
||||
cursor.execute(query, ["Games"]) |
||||
records = cursor.fetchall() |
||||
for row in records: |
||||
self.game_id = row[0] |
||||
cursor.close() |
||||
|
||||
else: |
||||
for row in records: |
||||
self.game_id = row[0] |
||||
self.populateGameAttrs() |
||||
|
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
finally: |
||||
if (sqliteConnection): |
||||
sqliteConnection.close() |
||||
print("The SQLite connection is closed") |
||||
|
||||
elif name is None and game_id is not None: |
||||
self.game_id = game_id |
||||
db = dbConnection() |
||||
query = "SELECT * FROM Games WHERE game_id=?" |
||||
values = [game_id, ] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
if records.__sizeof__() > 0: |
||||
self.populateGameAttrs() |
||||
db.closeDB() |
||||
|
||||
else: |
||||
print("No data given, initializing blank game object") |
||||
|
||||
def populateGameAttrs(self): |
||||
db = dbConnection() |
||||
query = "SELECT * FROM Games WHERE game_id=?" |
||||
values = [self.game_id] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
for row in records: |
||||
self.name = row[2] |
||||
self.rating = row[1] |
||||
self.description = row[3] |
||||
|
||||
query = "SELECT * FROM Game_On_Website WHERE game_id=?" |
||||
#print(self.game_id) |
||||
values = [self.game_id] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
for row in records: |
||||
self.downloadURLArr.append({'console_id': row[4], 'website_id': row[2], 'url': row[3]}) |
||||
self.downloadURL = row[3] |
||||
|
||||
query = "SELECT * FROM Game_On_Console WHERE game_id=?" |
||||
values = [self.game_id] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
for row in records: |
||||
self.console = row[2] |
||||
|
||||
return |
||||
|
||||
def updateGameAttrs(self, game_id=None): |
||||
if game_id is None: |
||||
print("Game ID is REQUIRED!") |
||||
return |
||||
else: |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
#print("Connected") |
||||
query = "SELECT * FROM Games WHERE game_id=?" |
||||
cursor.execute(query, [game_id, ]) |
||||
records = cursor.fetchall() |
||||
if len(records) > 0: |
||||
query = "UPDATE Games SET name=?, description=?, rating=?, thumbnail_uri=? WHERE game_id=?" |
||||
cursor.execute(query, [self.name, self.description, self.rating, self.thumbnail_uri, self.game_id]) |
||||
sqliteConnection.commit() |
||||
|
||||
else: |
||||
print("Game with this ID does not exist!") |
||||
cursor.close() |
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
return |
||||
|
||||
def addDownloadUrl(self, website, url, consoleId): |
||||
self.downloadURL = url |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
#print("Connected") |
||||
query = "SELECT * FROM Game_On_Website WHERE game_id=? AND url=?" |
||||
cursor.execute(query, [self.game_id, url]) |
||||
records = cursor.fetchall() |
||||
if len(records) > 0: |
||||
print("Game is already catalogued!") |
||||
else: |
||||
query = "INSERT INTO Game_On_Website (game_id, website_id, url, console_id) VALUES (?,?,?, ?)" |
||||
cursor.execute(query, [self.game_id, website.website_id, url, consoleId]) |
||||
sqliteConnection.commit() |
||||
cursor.close() |
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
return |
||||
|
||||
def addConsole(self, console): |
||||
self.console = console.name |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
#print("Connected") |
||||
query = "SELECT * FROM Game_On_Console WHERE game_id=? AND console_id=?" |
||||
cursor.execute(query, [self.game_id, console.console_id]) |
||||
records = cursor.fetchall() |
||||
if len(records) > 0: |
||||
print("Game is already catalogued!") |
||||
else: |
||||
query = "INSERT INTO Game_On_Console (game_id, console_id) VALUES (?,?)" |
||||
cursor.execute(query, [self.game_id, console.console_id]) |
||||
sqliteConnection.commit() |
||||
cursor.close() |
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
return |
||||
|
||||
def download(self, website, console): |
||||
session = HTMLSession() |
||||
for loc in self.downloadURLArr: |
||||
if loc['website_id'] == website.website_id and loc['console_id'] == console.console_id: |
||||
self.downloadURL = loc['url'] |
||||
|
||||
response = session.get(self.downloadURL) |
||||
|
||||
|
||||
|
||||
inputs = response.html.find('input') |
||||
for inputField in inputs: |
||||
if inputField.html.find('mediaId') != -1: |
||||
mediaID = re.findall('value=\"(.*?)\"', inputField.html)[0] |
||||
|
||||
headers = { |
||||
"User-Agent": 'Mozilla/5.0(X11; Linux x86_64; rv: 83.0) ' |
||||
'Gecko/20100101' |
||||
'Firefox / 83.0', |
||||
"Accept": 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', |
||||
"Accept-Language": 'en-US,en;q=0.5', |
||||
"Accept-Encoding": 'gzip, deflate, br', |
||||
"Connection": 'keep-alive', |
||||
"Referer": self.downloadURL, |
||||
"Cookie": '__cfduid = d186e9398ba52f9caf7e9a1fdae8b51511606415231', |
||||
"Upgrade-Insecure-Requests": '1', |
||||
} |
||||
serverNum = 4 |
||||
fileDownloaded = False |
||||
while not fileDownloaded: |
||||
try: |
||||
r = requests.get("https://download" + str(serverNum) + ".vimm.net/download/?mediaId=" + str(mediaID), |
||||
headers=headers) |
||||
|
||||
except requests.exceptions.RequestException as e: |
||||
serverNum -= 1 |
||||
print(e) |
||||
|
||||
else: |
||||
if r.status_code == 200: |
||||
pprint("saving " + self.name) |
||||
h = r.headers['content-disposition'] |
||||
saveLocation = os.path.expanduser('~/Downloads/RomScraper/') + console.name + "/" + self.name[0] + "/" + re.findall("filename=\"(.+)\"", h)[0] |
||||
os.makedirs(os.path.dirname(saveLocation), exist_ok=True) |
||||
with open(saveLocation, "w") as f: |
||||
f.write("FOOBAR") |
||||
fileDownloaded = True |
||||
else: |
||||
pprint("File not found on server" + str(serverNum) + ", trying next server") |
||||
serverNum -= 1 |
||||
sleep(2) |
||||
|
||||
if serverNum == 0: |
||||
pprint("Game was unable to be downloaded!") |
||||
fileDownloaded = True |
||||
return |
||||
@ -0,0 +1,77 @@
@@ -0,0 +1,77 @@
|
||||
import sqlite3 |
||||
from dbHandler import dbConnection |
||||
|
||||
|
||||
class Website: |
||||
website_id = 0 |
||||
name = "" |
||||
url = "" |
||||
description = "" |
||||
thumbnail_uri = "" |
||||
rating = 0 |
||||
|
||||
def __init__(self, website_id=None, name=None): |
||||
if website_id is None and name is not None: |
||||
self.name = name |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
print("Connected") |
||||
query = "SELECT * FROM Websites WHERE name=?" |
||||
cursor.execute(query, [name, ]) |
||||
records = cursor.fetchall() |
||||
if len(records) < 1: |
||||
print("Website is not in table") |
||||
query = "INSERT INTO Websites (name) VALUES (?)" |
||||
cursor.execute(query, [name, ]) |
||||
sqliteConnection.commit() |
||||
cursor.close() |
||||
else: |
||||
for row in records: |
||||
self.website_id = row[0] |
||||
self.url = row[2] |
||||
|
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
finally: |
||||
if (sqliteConnection): |
||||
sqliteConnection.close() |
||||
print("The SQLite connection is closed") |
||||
|
||||
elif name is None and website_id is not None: |
||||
db = dbConnection() |
||||
query = "SELECT * FROM Websites WHERE website_id=?" |
||||
values = [website_id, ] |
||||
records = db.queryDB(query, values, returnsRecords=True) |
||||
for row in records: |
||||
self.website_id = row[0] |
||||
self.url = row[2] |
||||
db.closeDB() |
||||
|
||||
else: |
||||
print("No data given, initializing blank Website object") |
||||
|
||||
def updateWebsiteAttrs(self, website_id=None): |
||||
if website_id is None: |
||||
print("Website ID is REQUIRED!") |
||||
return |
||||
else: |
||||
try: |
||||
sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
cursor = sqliteConnection.cursor() |
||||
print("Connected") |
||||
query = "SELECT * FROM Websites WHERE website_id=?" |
||||
cursor.execute(query, [website_id, ]) |
||||
records = cursor.fetchall() |
||||
if len(records) > 0: |
||||
print("Website is not in table") |
||||
query = "UPDATE Websites SET name=?, url=?, description=?, rating=?, thumbnail_uri=? WHERE website_id=? " |
||||
cursor.execute(query, [self.name, self.url, self.description, self.rating, self.thumbnail_uri, self.website_id]) |
||||
sqliteConnection.commit() |
||||
cursor.close() |
||||
else: |
||||
print("Website with this ID does not exist!") |
||||
|
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
@ -0,0 +1,19 @@
@@ -0,0 +1,19 @@
|
||||
[[source]] |
||||
name = "pypi" |
||||
url = "https://pypi.org/simple" |
||||
verify_ssl = true |
||||
|
||||
[dev-packages] |
||||
|
||||
[packages] |
||||
requests = "*" |
||||
requests-html = "*" |
||||
pandas = "*" |
||||
regex = "*" |
||||
cython = "*" |
||||
idna = "*" |
||||
websockets = "*" |
||||
pyinstaller = "*" |
||||
|
||||
[requires] |
||||
python_version = "3.9" |
||||
@ -0,0 +1,418 @@
@@ -0,0 +1,418 @@
|
||||
{ |
||||
"_meta": { |
||||
"hash": { |
||||
"sha256": "f4e9cb1c7f9504d32f1ee03f40a43fb2fb332366812306b244016dddac46d63d" |
||||
}, |
||||
"pipfile-spec": 6, |
||||
"requires": { |
||||
"python_version": "3.9" |
||||
}, |
||||
"sources": [ |
||||
{ |
||||
"name": "pypi", |
||||
"url": "https://pypi.org/simple", |
||||
"verify_ssl": true |
||||
} |
||||
] |
||||
}, |
||||
"default": { |
||||
"altgraph": { |
||||
"hashes": [ |
||||
"sha256:1f05a47122542f97028caf78775a095fbe6a2699b5089de8477eb583167d69aa", |
||||
"sha256:c623e5f3408ca61d4016f23a681b9adb100802ca3e3da5e718915a9e4052cebe" |
||||
], |
||||
"version": "==0.17" |
||||
}, |
||||
"appdirs": { |
||||
"hashes": [ |
||||
"sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", |
||||
"sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128" |
||||
], |
||||
"version": "==1.4.4" |
||||
}, |
||||
"beautifulsoup4": { |
||||
"hashes": [ |
||||
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35", |
||||
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25", |
||||
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666" |
||||
], |
||||
"version": "==4.9.3" |
||||
}, |
||||
"bs4": { |
||||
"hashes": [ |
||||
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" |
||||
], |
||||
"version": "==0.0.1" |
||||
}, |
||||
"certifi": { |
||||
"hashes": [ |
||||
"sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee", |
||||
"sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8" |
||||
], |
||||
"version": "==2021.5.30" |
||||
}, |
||||
"chardet": { |
||||
"hashes": [ |
||||
"sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", |
||||
"sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", |
||||
"version": "==4.0.0" |
||||
}, |
||||
"cssselect": { |
||||
"hashes": [ |
||||
"sha256:f612ee47b749c877ebae5bb77035d8f4202c6ad0f0fc1271b3c18ad6c4468ecf", |
||||
"sha256:f95f8dedd925fd8f54edb3d2dfb44c190d9d18512377d3c1e2388d16126879bc" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", |
||||
"version": "==1.1.0" |
||||
}, |
||||
"cython": { |
||||
"hashes": [ |
||||
"sha256:0c4b9f7e3aa004cf3f364e3e772f55fec5740485bafea99d1f13bdc9bbd8a545", |
||||
"sha256:20402ef316393168909926ab21848aa6e08e39bed5003b657139774e66166cd0", |
||||
"sha256:20cb50d9fede8029bdb50875458f07a27f909289aeed4cdb9c19544dd9a9bc45", |
||||
"sha256:2365f3b5e6451b6bc6dcd262230656f4ade1d862ec2f6c22154deebef37c08b6", |
||||
"sha256:266459c7e48fe3c6c492b297e4033e42d4c6863cc1a1ff7cc4034949fc574fa6", |
||||
"sha256:282263628c5d601b313d5920f7b6d7e08c7fedbddacd080c4858aa04d86b6b4b", |
||||
"sha256:2a3bbce689a2fddb85aa66712d93875c99bf7f64ac82b1d149ecce522a7a4e0c", |
||||
"sha256:2af52d312e96b38ded38b34d06e22685c226b1b0e58278bd27209f5d2385d115", |
||||
"sha256:355a6e768d91e21fbf477b61881bab64b7a2da386a166898997bccefd532cf5d", |
||||
"sha256:37ff66039e3d138ec968ee1d1e12441fa5fb4e6a9c5458bc3c3a232f01be4a7d", |
||||
"sha256:3b29224eb62309a10819d923dc6262f769e4f3facfee3cd06372c355e5b38b33", |
||||
"sha256:3ef530f975e3a760e7282fce2a25f900fa63f96d17321b4aa5f5542eb9859cdf", |
||||
"sha256:41cd0dd2ff5d78466e73409db509887a84449b400074d4f217980cedbb18e4be", |
||||
"sha256:474c1a29ab43e29d990df279e2cf6aa96baa9208f5cd4bc76ac87ffcdf1e2945", |
||||
"sha256:4858043ac5f96a8f0277cf63760bb39b9521c1f897678cf1d22423f3e758f4ed", |
||||
"sha256:4b0bcf2e06a9063fc78c3243ed4003228375d532ef13b9e5d7183be8f0a52cf5", |
||||
"sha256:4b6824b58d4373224fc76ee8bee6b35c2d17c91a1ed0fa67b88440f63daebe50", |
||||
"sha256:4d7c3b0882d8757c601eaf288fc0d321d5c7ac6c3afb8c42eddf9325a3419cf5", |
||||
"sha256:519fccf526d26b377e1db22f22aa44889b28bc5833ec106588cb13557e8ba2da", |
||||
"sha256:58dc06871bfdb0592542d779714fe9f918e11ba20ac07757dd63b198bdc704fe", |
||||
"sha256:5a6792153b728a0240e55bbb5b643f4f7e45c76319e03abf15bf367471ea1d1a", |
||||
"sha256:5be3ae3189cf7d0e9bbeafb854496dc7030c6f6a5602d809435fab8223543a41", |
||||
"sha256:625a16103770fd92b487b701fb0c07e5790b080f40fa11ce572a2d56d9e9fcca", |
||||
"sha256:6a0d31452f0245daacb14c979c77e093eb1a546c760816b5eed0047686baad8e", |
||||
"sha256:794e3df0b57e16bce7583ac909126f4cb381fe566adadb20484d89095855eedb", |
||||
"sha256:7b7a766726d207d7cd57aff0fcb4b35ce042d3cc88a421fcdb45eeb61a5b9d12", |
||||
"sha256:7d6a33c8a11f05f698e215bfdb837f32c27f63c20f3af863557ed91c748dc2be", |
||||
"sha256:a8eed9c82e8fe07b8a8ffbd36018871a17458903fc25c9d015f37b54513a3efd", |
||||
"sha256:aa3bb0928fb2aa3a8828801eb8b29af2261c199f805ae835467489e2bdd00372", |
||||
"sha256:b0699f0dc90181f2458fdb8170455e7798a309e18f41379eda7a2dc8c7aadee0", |
||||
"sha256:c4b82461edbbcf90f19b319006345b77474a2d7514e1476d49a14bbd55d6b797", |
||||
"sha256:ceccc03b633113ede1f14ad914a6db5c278ce108c8ddb308a5c01c1567d8a02a", |
||||
"sha256:ef21c51350462160456eb71df31b0869e5141e940f22c61c358bdb6e3ebc3388", |
||||
"sha256:f4aca6bffb1c1c3c4ada3347d0b162a699c18a66e097ee08b63b3a35118fdfcc", |
||||
"sha256:ff885f18d169759b57f116d3956e45cd2b9cba989fde348bba091544c668dc11" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==0.29.23" |
||||
}, |
||||
"fake-useragent": { |
||||
"hashes": [ |
||||
"sha256:c104998b750eb097eefc28ae28e92d66397598d2cf41a31aa45d5559ef1adf35" |
||||
], |
||||
"version": "==0.1.11" |
||||
}, |
||||
"idna": { |
||||
"hashes": [ |
||||
"sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", |
||||
"sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==2.10" |
||||
}, |
||||
"lxml": { |
||||
"hashes": [ |
||||
"sha256:079f3ae844f38982d156efce585bc540c16a926d4436712cf4baee0cce487a3d", |
||||
"sha256:0fbcf5565ac01dff87cbfc0ff323515c823081c5777a9fc7703ff58388c258c3", |
||||
"sha256:122fba10466c7bd4178b07dba427aa516286b846b2cbd6f6169141917283aae2", |
||||
"sha256:1b38116b6e628118dea5b2186ee6820ab138dbb1e24a13e478490c7db2f326ae", |
||||
"sha256:1b7584d421d254ab86d4f0b13ec662a9014397678a7c4265a02a6d7c2b18a75f", |
||||
"sha256:26e761ab5b07adf5f555ee82fb4bfc35bf93750499c6c7614bd64d12aaa67927", |
||||
"sha256:289e9ca1a9287f08daaf796d96e06cb2bc2958891d7911ac7cae1c5f9e1e0ee3", |
||||
"sha256:2a9d50e69aac3ebee695424f7dbd7b8c6d6eb7de2a2eb6b0f6c7db6aa41e02b7", |
||||
"sha256:3082c518be8e97324390614dacd041bb1358c882d77108ca1957ba47738d9d59", |
||||
"sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f", |
||||
"sha256:3439c71103ef0e904ea0a1901611863e51f50b5cd5e8654a151740fde5e1cade", |
||||
"sha256:36108c73739985979bf302006527cf8a20515ce444ba916281d1c43938b8bb96", |
||||
"sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468", |
||||
"sha256:4289728b5e2000a4ad4ab8da6e1db2e093c63c08bdc0414799ee776a3f78da4b", |
||||
"sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4", |
||||
"sha256:4c61b3a0db43a1607d6264166b230438f85bfed02e8cff20c22e564d0faff354", |
||||
"sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83", |
||||
"sha256:5a0a14e264069c03e46f926be0d8919f4105c1623d620e7ec0e612a2e9bf1c04", |
||||
"sha256:5c8c163396cc0df3fd151b927e74f6e4acd67160d6c33304e805b84293351d16", |
||||
"sha256:66e575c62792c3f9ca47cb8b6fab9e35bab91360c783d1606f758761810c9791", |
||||
"sha256:6f12e1427285008fd32a6025e38e977d44d6382cf28e7201ed10d6c1698d2a9a", |
||||
"sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51", |
||||
"sha256:7610b8c31688f0b1be0ef882889817939490a36d0ee880ea562a4e1399c447a1", |
||||
"sha256:76fa7b1362d19f8fbd3e75fe2fb7c79359b0af8747e6f7141c338f0bee2f871a", |
||||
"sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f", |
||||
"sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee", |
||||
"sha256:820628b7b3135403540202e60551e741f9b6d3304371712521be939470b454ec", |
||||
"sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969", |
||||
"sha256:89b8b22a5ff72d89d48d0e62abb14340d9e99fd637d046c27b8b257a01ffbe28", |
||||
"sha256:92e821e43ad382332eade6812e298dc9701c75fe289f2a2d39c7960b43d1e92a", |
||||
"sha256:b007cbb845b28db4fb8b6a5cdcbf65bacb16a8bd328b53cbc0698688a68e1caa", |
||||
"sha256:bc4313cbeb0e7a416a488d72f9680fffffc645f8a838bd2193809881c67dd106", |
||||
"sha256:bccbfc27563652de7dc9bdc595cb25e90b59c5f8e23e806ed0fd623755b6565d", |
||||
"sha256:c47ff7e0a36d4efac9fd692cfa33fbd0636674c102e9e8d9b26e1b93a94e7617", |
||||
"sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4", |
||||
"sha256:cdaf11d2bd275bf391b5308f86731e5194a21af45fbaaaf1d9e8147b9160ea92", |
||||
"sha256:ce256aaa50f6cc9a649c51be3cd4ff142d67295bfc4f490c9134d0f9f6d58ef0", |
||||
"sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4", |
||||
"sha256:d916d31fd85b2f78c76400d625076d9124de3e4bda8b016d25a050cc7d603f24", |
||||
"sha256:df7c53783a46febb0e70f6b05df2ba104610f2fb0d27023409734a3ecbb78fb2", |
||||
"sha256:e1cbd3f19a61e27e011e02f9600837b921ac661f0c40560eefb366e4e4fb275e", |
||||
"sha256:efac139c3f0bf4f0939f9375af4b02c5ad83a622de52d6dfa8e438e8e01d0eb0", |
||||
"sha256:efd7a09678fd8b53117f6bae4fa3825e0a22b03ef0a932e070c0bdbb3a35e654", |
||||
"sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2", |
||||
"sha256:f8380c03e45cf09f8557bdaa41e1fa7c81f3ae22828e1db470ab2a6c96d8bc23", |
||||
"sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", |
||||
"version": "==4.6.3" |
||||
}, |
||||
"numpy": { |
||||
"hashes": [ |
||||
"sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010", |
||||
"sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd", |
||||
"sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43", |
||||
"sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9", |
||||
"sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df", |
||||
"sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400", |
||||
"sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2", |
||||
"sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4", |
||||
"sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a", |
||||
"sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6", |
||||
"sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8", |
||||
"sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b", |
||||
"sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8", |
||||
"sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb", |
||||
"sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2", |
||||
"sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f", |
||||
"sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4", |
||||
"sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a", |
||||
"sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16", |
||||
"sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f", |
||||
"sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69", |
||||
"sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65", |
||||
"sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17", |
||||
"sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48" |
||||
], |
||||
"markers": "python_version >= '3.7'", |
||||
"version": "==1.20.3" |
||||
}, |
||||
"pandas": { |
||||
"hashes": [ |
||||
"sha256:167693a80abc8eb28051fbd184c1b7afd13ce2c727a5af47b048f1ea3afefff4", |
||||
"sha256:2111c25e69fa9365ba80bbf4f959400054b2771ac5d041ed19415a8b488dc70a", |
||||
"sha256:298f0553fd3ba8e002c4070a723a59cdb28eda579f3e243bc2ee397773f5398b", |
||||
"sha256:2b063d41803b6a19703b845609c0b700913593de067b552a8b24dd8eeb8c9895", |
||||
"sha256:2cb7e8f4f152f27dc93f30b5c7a98f6c748601ea65da359af734dd0cf3fa733f", |
||||
"sha256:52d2472acbb8a56819a87aafdb8b5b6d2b3386e15c95bde56b281882529a7ded", |
||||
"sha256:612add929bf3ba9d27b436cc8853f5acc337242d6b584203f207e364bb46cb12", |
||||
"sha256:649ecab692fade3cbfcf967ff936496b0cfba0af00a55dfaacd82bdda5cb2279", |
||||
"sha256:68d7baa80c74aaacbed597265ca2308f017859123231542ff8a5266d489e1858", |
||||
"sha256:8d4c74177c26aadcfb4fd1de6c1c43c2bf822b3e0fc7a9b409eeaf84b3e92aaa", |
||||
"sha256:971e2a414fce20cc5331fe791153513d076814d30a60cd7348466943e6e909e4", |
||||
"sha256:9db70ffa8b280bb4de83f9739d514cd0735825e79eef3a61d312420b9f16b758", |
||||
"sha256:b730add5267f873b3383c18cac4df2527ac4f0f0eed1c6cf37fcb437e25cf558", |
||||
"sha256:bd659c11a4578af740782288cac141a322057a2e36920016e0fc7b25c5a4b686", |
||||
"sha256:c601c6fdebc729df4438ec1f62275d6136a0dd14d332fc0e8ce3f7d2aadb4dd6", |
||||
"sha256:d0877407359811f7b853b548a614aacd7dea83b0c0c84620a9a643f180060950" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==1.2.4" |
||||
}, |
||||
"parse": { |
||||
"hashes": [ |
||||
"sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b" |
||||
], |
||||
"version": "==1.19.0" |
||||
}, |
||||
"pyee": { |
||||
"hashes": [ |
||||
"sha256:383973b63ad7ed5e3c0311f8b179c52981f9e7b3eaea0e9a830d13ec34dde65f", |
||||
"sha256:92dacc5bd2bdb8f95aa8dd2585d47ca1c4840e2adb95ccf90034d64f725bfd31" |
||||
], |
||||
"version": "==8.1.0" |
||||
}, |
||||
"pyinstaller": { |
||||
"hashes": [ |
||||
"sha256:5ecf8bbc230d7298a796e52bb745b95eee12878d141f1645612c99246ecd23f2" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==4.3" |
||||
}, |
||||
"pyinstaller-hooks-contrib": { |
||||
"hashes": [ |
||||
"sha256:27558072021857d89524c42136feaa2ffe4f003f1bdf0278f9b24f6902c1759c", |
||||
"sha256:892310e6363655838485ee748bf1c5e5cade7963686d9af8650ee218a3e0b031" |
||||
], |
||||
"version": "==2021.1" |
||||
}, |
||||
"pyppeteer": { |
||||
"hashes": [ |
||||
"sha256:c2974be1afa13b17f7ecd120d265d8b8cd324d536a231c3953ca872b68aba4af", |
||||
"sha256:d4cb4a5ef94b00c1073aed888b39646ce26cff3339cff7a3f1f1cc307bf50408" |
||||
], |
||||
"markers": "python_version < '4' and python_full_version >= '3.6.1'", |
||||
"version": "==0.2.5" |
||||
}, |
||||
"pyquery": { |
||||
"hashes": [ |
||||
"sha256:1fc33b7699455ed25c75282bc8f80ace1ac078b0dda5a933dacbd8b1c1f83963", |
||||
"sha256:a388eefb6bc4a55350de0316fbd97cda999ae669b6743ae5b99102ba54f5aa72" |
||||
], |
||||
"version": "==1.4.3" |
||||
}, |
||||
"python-dateutil": { |
||||
"hashes": [ |
||||
"sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c", |
||||
"sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", |
||||
"version": "==2.8.1" |
||||
}, |
||||
"pytz": { |
||||
"hashes": [ |
||||
"sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da", |
||||
"sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798" |
||||
], |
||||
"version": "==2021.1" |
||||
}, |
||||
"regex": { |
||||
"hashes": [ |
||||
"sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5", |
||||
"sha256:11d773d75fa650cd36f68d7ca936e3c7afaae41b863b8c387a22aaa78d3c5c79", |
||||
"sha256:18c071c3eb09c30a264879f0d310d37fe5d3a3111662438889ae2eb6fc570c31", |
||||
"sha256:1e1c20e29358165242928c2de1482fb2cf4ea54a6a6dea2bd7a0e0d8ee321500", |
||||
"sha256:281d2fd05555079448537fe108d79eb031b403dac622621c78944c235f3fcf11", |
||||
"sha256:314d66636c494ed9c148a42731b3834496cc9a2c4251b1661e40936814542b14", |
||||
"sha256:32e65442138b7b76dd8173ffa2cf67356b7bc1768851dded39a7a13bf9223da3", |
||||
"sha256:339456e7d8c06dd36a22e451d58ef72cef293112b559010db3d054d5560ef439", |
||||
"sha256:3916d08be28a1149fb97f7728fca1f7c15d309a9f9682d89d79db75d5e52091c", |
||||
"sha256:3a9cd17e6e5c7eb328517969e0cb0c3d31fd329298dd0c04af99ebf42e904f82", |
||||
"sha256:47bf5bf60cf04d72bf6055ae5927a0bd9016096bf3d742fa50d9bf9f45aa0711", |
||||
"sha256:4c46e22a0933dd783467cf32b3516299fb98cfebd895817d685130cc50cd1093", |
||||
"sha256:4c557a7b470908b1712fe27fb1ef20772b78079808c87d20a90d051660b1d69a", |
||||
"sha256:52ba3d3f9b942c49d7e4bc105bb28551c44065f139a65062ab7912bef10c9afb", |
||||
"sha256:563085e55b0d4fb8f746f6a335893bda5c2cef43b2f0258fe1020ab1dd874df8", |
||||
"sha256:598585c9f0af8374c28edd609eb291b5726d7cbce16be6a8b95aa074d252ee17", |
||||
"sha256:619d71c59a78b84d7f18891fe914446d07edd48dc8328c8e149cbe0929b4e000", |
||||
"sha256:67bdb9702427ceddc6ef3dc382455e90f785af4c13d495f9626861763ee13f9d", |
||||
"sha256:6d1b01031dedf2503631d0903cb563743f397ccaf6607a5e3b19a3d76fc10480", |
||||
"sha256:741a9647fcf2e45f3a1cf0e24f5e17febf3efe8d4ba1281dcc3aa0459ef424dc", |
||||
"sha256:7c2a1af393fcc09e898beba5dd59196edaa3116191cc7257f9224beaed3e1aa0", |
||||
"sha256:7d9884d86dd4dd489e981d94a65cd30d6f07203d90e98f6f657f05170f6324c9", |
||||
"sha256:90f11ff637fe8798933fb29f5ae1148c978cccb0452005bf4c69e13db951e765", |
||||
"sha256:919859aa909429fb5aa9cf8807f6045592c85ef56fdd30a9a3747e513db2536e", |
||||
"sha256:96fcd1888ab4d03adfc9303a7b3c0bd78c5412b2bfbe76db5b56d9eae004907a", |
||||
"sha256:97f29f57d5b84e73fbaf99ab3e26134e6687348e95ef6b48cfd2c06807005a07", |
||||
"sha256:980d7be47c84979d9136328d882f67ec5e50008681d94ecc8afa8a65ed1f4a6f", |
||||
"sha256:a91aa8619b23b79bcbeb37abe286f2f408d2f2d6f29a17237afda55bb54e7aac", |
||||
"sha256:ade17eb5d643b7fead300a1641e9f45401c98eee23763e9ed66a43f92f20b4a7", |
||||
"sha256:b9c3db21af35e3b3c05764461b262d6f05bbca08a71a7849fd79d47ba7bc33ed", |
||||
"sha256:bd28bc2e3a772acbb07787c6308e00d9626ff89e3bfcdebe87fa5afbfdedf968", |
||||
"sha256:bf5824bfac591ddb2c1f0a5f4ab72da28994548c708d2191e3b87dd207eb3ad7", |
||||
"sha256:c0502c0fadef0d23b128605d69b58edb2c681c25d44574fc673b0e52dce71ee2", |
||||
"sha256:c38c71df845e2aabb7fb0b920d11a1b5ac8526005e533a8920aea97efb8ec6a4", |
||||
"sha256:ce15b6d103daff8e9fee13cf7f0add05245a05d866e73926c358e871221eae87", |
||||
"sha256:d3029c340cfbb3ac0a71798100ccc13b97dddf373a4ae56b6a72cf70dfd53bc8", |
||||
"sha256:e512d8ef5ad7b898cdb2d8ee1cb09a8339e4f8be706d27eaa180c2f177248a10", |
||||
"sha256:e8e5b509d5c2ff12f8418006d5a90e9436766133b564db0abaec92fd27fcee29", |
||||
"sha256:ee54ff27bf0afaf4c3b3a62bcd016c12c3fdb4ec4f413391a90bd38bc3624605", |
||||
"sha256:fa4537fb4a98fe8fde99626e4681cc644bdcf2a795038533f9f711513a862ae6", |
||||
"sha256:fd45ff9293d9274c5008a2054ecef86a9bfe819a67c7be1afb65e69b405b3042" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==2021.4.4" |
||||
}, |
||||
"requests": { |
||||
"hashes": [ |
||||
"sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", |
||||
"sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==2.25.1" |
||||
}, |
||||
"requests-html": { |
||||
"hashes": [ |
||||
"sha256:7e929ecfed95fb1d0994bb368295d6d7c4d06b03fcb900c33d7d0b17e6003947", |
||||
"sha256:cb8a78cf829c4eca9d6233f28524f65dd2bfaafb4bdbbc407f0a0b8f487df6e2" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==0.10.0" |
||||
}, |
||||
"six": { |
||||
"hashes": [ |
||||
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", |
||||
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", |
||||
"version": "==1.16.0" |
||||
}, |
||||
"soupsieve": { |
||||
"hashes": [ |
||||
"sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc", |
||||
"sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b" |
||||
], |
||||
"markers": "python_version >= '3.0'", |
||||
"version": "==2.2.1" |
||||
}, |
||||
"tqdm": { |
||||
"hashes": [ |
||||
"sha256:24be966933e942be5f074c29755a95b315c69a91f839a29139bf26ffffe2d3fd", |
||||
"sha256:aa0c29f03f298951ac6318f7c8ce584e48fa22ec26396e6411e43d038243bdb2" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", |
||||
"version": "==4.61.1" |
||||
}, |
||||
"urllib3": { |
||||
"hashes": [ |
||||
"sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c", |
||||
"sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098" |
||||
], |
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", |
||||
"version": "==1.26.5" |
||||
}, |
||||
"w3lib": { |
||||
"hashes": [ |
||||
"sha256:0161d55537063e00d95a241663ede3395c4c6d7b777972ba2fd58bbab2001e53", |
||||
"sha256:0ad6d0203157d61149fd45aaed2e24f53902989c32fc1dccc2e2bfba371560df" |
||||
], |
||||
"version": "==1.22.0" |
||||
}, |
||||
"websockets": { |
||||
"hashes": [ |
||||
"sha256:0e4fb4de42701340bd2353bb2eee45314651caa6ccee80dbd5f5d5978888fed5", |
||||
"sha256:1d3f1bf059d04a4e0eb4985a887d49195e15ebabc42364f4eb564b1d065793f5", |
||||
"sha256:20891f0dddade307ffddf593c733a3fdb6b83e6f9eef85908113e628fa5a8308", |
||||
"sha256:295359a2cc78736737dd88c343cd0747546b2174b5e1adc223824bcaf3e164cb", |
||||
"sha256:2db62a9142e88535038a6bcfea70ef9447696ea77891aebb730a333a51ed559a", |
||||
"sha256:3762791ab8b38948f0c4d281c8b2ddfa99b7e510e46bd8dfa942a5fff621068c", |
||||
"sha256:3db87421956f1b0779a7564915875ba774295cc86e81bc671631379371af1170", |
||||
"sha256:3ef56fcc7b1ff90de46ccd5a687bbd13a3180132268c4254fc0fa44ecf4fc422", |
||||
"sha256:4f9f7d28ce1d8f1295717c2c25b732c2bc0645db3215cf757551c392177d7cb8", |
||||
"sha256:5c01fd846263a75bc8a2b9542606927cfad57e7282965d96b93c387622487485", |
||||
"sha256:5c65d2da8c6bce0fca2528f69f44b2f977e06954c8512a952222cea50dad430f", |
||||
"sha256:751a556205d8245ff94aeef23546a1113b1dd4f6e4d102ded66c39b99c2ce6c8", |
||||
"sha256:7ff46d441db78241f4c6c27b3868c9ae71473fe03341340d2dfdbe8d79310acc", |
||||
"sha256:965889d9f0e2a75edd81a07592d0ced54daa5b0785f57dc429c378edbcffe779", |
||||
"sha256:9b248ba3dd8a03b1a10b19efe7d4f7fa41d158fdaa95e2cf65af5a7b95a4f989", |
||||
"sha256:9bef37ee224e104a413f0780e29adb3e514a5b698aabe0d969a6ba426b8435d1", |
||||
"sha256:c1ec8db4fac31850286b7cd3b9c0e1b944204668b8eb721674916d4e28744092", |
||||
"sha256:c8a116feafdb1f84607cb3b14aa1418424ae71fee131642fc568d21423b51824", |
||||
"sha256:ce85b06a10fc65e6143518b96d3dca27b081a740bae261c2fb20375801a9d56d", |
||||
"sha256:d705f8aeecdf3262379644e4b55107a3b55860eb812b673b28d0fbc347a60c55", |
||||
"sha256:e898a0863421650f0bebac8ba40840fc02258ef4714cb7e1fd76b6a6354bda36", |
||||
"sha256:f8a7bff6e8664afc4e6c28b983845c5bc14965030e3fb98789734d416af77c4b" |
||||
], |
||||
"index": "pypi", |
||||
"version": "==8.1" |
||||
} |
||||
}, |
||||
"develop": {} |
||||
} |
||||
Binary file not shown.
@ -0,0 +1,60 @@
@@ -0,0 +1,60 @@
|
||||
import sqlite3 |
||||
|
||||
|
||||
class dbConnection: |
||||
dbname = 'Romscraper.db' |
||||
|
||||
def __init__(self, dbname=None): |
||||
if dbname is not None: |
||||
self.dbname = dbname |
||||
|
||||
try: |
||||
self.sqliteConnection = sqlite3.connect('Romscraper.db') |
||||
self.cursor = self.sqliteConnection.cursor() |
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
#finally: |
||||
#print("Database is initialized") |
||||
|
||||
def queryDB(self, query=None, values=None, commit=None, returnsRecords=None): |
||||
|
||||
if query is None: |
||||
print("This needs a query!") |
||||
return |
||||
else: |
||||
try: |
||||
if values is not None: |
||||
self.cursor.execute(query, values) |
||||
else: |
||||
self.cursor.execute(query) |
||||
|
||||
except sqlite3.Error as error: |
||||
print("Failed to fetch data from database: ", error) |
||||
|
||||
finally: |
||||
if commit is True: |
||||
self.sqliteConnection.commit() |
||||
if returnsRecords is True: |
||||
records = self.cursor.fetchall() |
||||
return records |
||||
|
||||
|
||||
def closeDB(self, commit=None): |
||||
if commit is None: |
||||
commit = False |
||||
|
||||
if commit: |
||||
self.sqliteConnection.commit() |
||||
self.sqliteConnection.close() |
||||
return |
||||
|
||||
def searchByName(self, table, name, fuzzymatch): |
||||
if fuzzymatch: |
||||
name = "%" + name + "%" |
||||
query = "SELECT * FROM " + table + " WHERE name LIKE ?" |
||||
else: |
||||
query = "SELECT * FROM " + table + " WHERE name = ?" |
||||
|
||||
self.cursor.execute(query, [name]) |
||||
records = self.cursor.fetchall() |
||||
return records |
||||
@ -0,0 +1,94 @@
@@ -0,0 +1,94 @@
|
||||
import os |
||||
from random import random |
||||
import requests |
||||
from requests_html import HTMLSession |
||||
from time import sleep |
||||
import re |
||||
from pprint import pprint |
||||
from Models.Website import Website |
||||
from Models.Console import Console |
||||
from Models.Game import Game |
||||
from dbHandler import dbConnection |
||||
|
||||
site = Website(website_id=1) |
||||
|
||||
|
||||
def searchGames(gamename, consolename="", fuzzyMatch=True): |
||||
db = dbConnection() |
||||
games = db.searchByName("Games", gamename, fuzzyMatch) |
||||
return games |
||||
|
||||
|
||||
def indexGame(gameId, website): |
||||
print(website.url) |
||||
url = website.url + str(gameId) |
||||
gamefound = False |
||||
try: |
||||
session = HTMLSession() |
||||
response = session.get(url) |
||||
if len(response.html.find('h2')) > 0: |
||||
# This part gets the game info |
||||
console = Console(name=response.html.find('h2')[0].text.split('\n')[0]) |
||||
game = Game(name=response.html.find('h2')[0].text.split('\n')[1]) |
||||
game.addDownloadUrl(site, url, console.console_id) |
||||
game.addConsole(console) |
||||
print("GameID: " + str(game.game_id)) |
||||
print("Game Name: " + game.name) |
||||
print("Game Console: " + game.console) |
||||
print("Download URL: " + game.downloadURL) |
||||
print("ConsoleID: " + str(console.console_id)) |
||||
gamefound = True |
||||
else: |
||||
print("There is no game with this ID") |
||||
except requests.exceptions.RequestException as e: |
||||
print(e) |
||||
|
||||
return gamefound |
||||
|
||||
|
||||
# gameID = 25200 |
||||
# consecutiveFailedURLs = 0 |
||||
# while consecutiveFailedURLs < 20: |
||||
# if not indexGame(gameID, website=Website(website_id=1)): |
||||
# consecutiveFailedURLs += 1 |
||||
# else: |
||||
# consecutiveFailedURLs = 0 |
||||
# gameID += 1 |
||||
# sleep(random()) |
||||
# if gameID % 200 == 0: |
||||
# sleep(random()*100) |
||||
# |
||||
# print("Last ID: " + gameID) |
||||
|
||||
|
||||
# game = Game(game_id=483) |
||||
# print(game.name) |
||||
# game.download(site, Console(console_id=1)) |
||||
|
||||
print("Enter a game name (full or partial) that you'd like to search for:") |
||||
searchterm = input() |
||||
games = searchGames(searchterm, True) |
||||
gamedownloadarr = [] |
||||
for game in games: |
||||
gameobj = Game(game[0]) |
||||
# print("\n\n"+gameobj.name + "\n__________________________\n") |
||||
|
||||
for row in gameobj.downloadURLArr: |
||||
gamedownloadarr.append({"game": gameobj.name, 'game_id': gameobj.game_id, "console_id": row['console_id'], "url": row['url']}) |
||||
|
||||
i = 1 |
||||
for game in gamedownloadarr: |
||||
print(str(i) + ") " + game['game'] + " - " + Console(console_id=game['console_id']).name + ": " + game['url']) |
||||
i += 1 |
||||
|
||||
print("What would you like to download?") |
||||
print("You can download multiple by typing multiple numbers with spaces between, ex: \"1 2 3\"...") |
||||
downloadSelRaw = input() |
||||
downloadlist = downloadSelRaw.split() |
||||
for selection in downloadlist: |
||||
selection = int(selection) |
||||
gamedownload = Game(game_id=gamedownloadarr[selection-1]['game_id']) |
||||
gamedownload.download(site, Console(gamedownloadarr[selection-1]['console_id'])) |
||||
print("Waiting so server doesn't block our requests.....") |
||||
sleep(10) |
||||
|
||||
Loading…
Reference in new issue