import requests import json import os import time from bs4 import BeautifulSoup from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support.expected_conditions import presence_of_element_located from selenium.webdriver.firefox.options import Options options = Options() options.headless = True base_url = "https://www.partnersinrhyme.com/royaltyfreemusic" base_files_url = "https://www.partnersinrhyme.com/files/" base_out_url = "files/" def getCollection(collection_url): r = requests.get(collection_url) soup = BeautifulSoup(r.text, 'html.parser') player = soup.iframe["src"]; p = requests.get("http:" + player); with webdriver.Firefox(options=options) as driver: driver.get("http:" + player) el = WebDriverWait(driver, 60).until(f) time.sleep(1) psoup = BeautifulSoup(driver.page_source, 'html.parser') driver.quit() for li in psoup.find_all("li"): print("downloading...", li.attrs['data-mp3']) downloadSong(base_files_url + li.attrs['data-mp3']) def downloadSong(song_url): (collection_name, outFile) = getSongFromURL(song_url); outDir = base_out_url + collection_name if not os.path.exists(outDir): os.makedirs(outDir) i = requests.get(song_url) with open(os.path.join(outDir, outFile), 'wb') as tempFile: tempFile.write(i.content) def getSongFromURL(song_url): list = song_url.split("/") print(list) return (list[4], list[-1]) def f(d): return d.find_element_by_class_name("listContainer")