import requests, time, re, os, configparser, sys, argparse, shutil from bs4 import BeautifulSoup def main(args: dict) -> None: print(f"Fetching PDFs.") if not os.path.isdir("pdfs"): print("Setting up PDFs directory.") os.mkdir("pdfs") if args.save_pages and not os.path.isdir("html"): print("Setting up HTML directory.") os.mkdir("html") config = configparser.ConfigParser(interpolation=None) if not os.path.isfile('config.ini'): print("config.ini is missing!") sys.exit(0) config.read('config.ini') if "username" not in config["DEFAULT"] or "password" not in config["DEFAULT"] or config["DEFAULT"]["username"] == "" or config["DEFAULT"]["password"] == "": print("username and password must be present in config.ini.") sys.exit(0) INFOVAYA_ROOT = "https://events.infovaya.com/" info = { "username": config["DEFAULT"]["username"], "password": config["DEFAULT"]["password"], "action": "login" } infovaya_session = requests.Session() infovaya_creds_request = infovaya_session.post(f"{INFOVAYA_ROOT}login", data=info) infovaya_creds_request.raise_for_status() sessions_request = infovaya_session.get("{}event?id=88&actionMenu=sessions".format(INFOVAYA_ROOT)) sessions_request.raise_for_status() sessions_page_soup = BeautifulSoup(sessions_request.text, "html.parser") if args.save_pages: with open("html/main.html", "w+", encoding="utf-8") as file: file.write(sessions_page_soup.prettify()) print("Starting to search for PDFs. This may take a while...") for session_link in sessions_page_soup.find_all("a", href=re.compile("session\?id")): session_href = session_link["href"][1:] page_name = f"session_{session_href.split('=')[-1]}.html" if not os.path.isfile(f"html/{page_name}"): time.sleep(1) print(f"Fetching {session_href} ... ", end="") page_request = infovaya_session.get(f"{INFOVAYA_ROOT}{session_href}") page_request.raise_for_status() print("Success! Finding presentations... ", end="") page_soup = BeautifulSoup(page_request.text, "html.parser") if args.save_pages: with open(f"html/{page_name}", "w+", encoding="utf-8") as file: file.write(page_soup.prettify()) print(f"Wrote to file html/{page_name}.") else: print() # import pdb; pdb.set_trace() for presentation_link in page_soup.find_all("a", href=re.compile("presentation\?id")): presentation_href = presentation_link["href"][1:] pres_page_name = f"pres_{presentation_href.split('=')[-1]}.html" if not os.path.isfile(f"html/{pres_page_name}"): print(f" Fetching {presentation_href} ...", end="") pres_request = infovaya_session.get(f"{INFOVAYA_ROOT}{presentation_href}") pres_request.raise_for_status() print("Success! ", end="") pres_page_soup = BeautifulSoup(pres_request.text, "html.parser") if args.save_pages: with open(f"html/{pres_page_name}", "w+", encoding="utf-8") as file: file.write(pres_page_soup.prettify()) pres_page_title = pres_page_soup.h1.string print(f"Found '{pres_page_title}'. Finding PDF...") for pdf_tag in pres_page_soup.find_all("a", href=re.compile("pdfviewer")): pdf_name = pdf_tag["href"].split("/")[-1] file_name = f"{pres_page_title.replace(' ', '_')}-{pdf_name}" pdf_request = infovaya_session.get(f"{INFOVAYA_ROOT}{pdf_tag['href'][1:]}") pdf_request.raise_for_status() print(f"Fetching PDF for '{pres_page_title}'...") with open(f"pdfs/{file_name}", "wb") as file: file.write(pdf_request.content) print(f"Saved as pdfs/{file_name}") time.sleep(2) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-s", "--save-pages", action="store_true", help="Save HTML of ") # parser.add_argument("-i", "--images", action="store_true", help="Set this flag to download images as well as HTML.\nNOTE: This may be VERY bandwidth and disk intensive!") args = parser.parse_args() print(args) main(args)