from pyvirtualdisplay import Display from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import urllib.request from bs4 import BeautifulSoup page=BeautifulSoup(urllib.request.urlopen("https://www.crowd2fund.com/exchange").read(), 'html.parser') totalItems=int(page.select_one("span[class=total-items]").text) #totalItems=30 maxPage=totalItems/15 pages=[str(i) for i in range(1,int(maxPage) + 1)] soups = [] for i in pages: soup=BeautifulSoup(urllib.request.urlopen("https://www.crowd2fund.com/exchange/"+i+"/?type=&status[]=good&sort=&direction=&perPage=15").read(), 'html.parser') soup2=soup.find_all(lambda tag: tag.name == 'a' and tag.get('class') == ['btn-remote-modal']) soups.extend(soup2) refs=[] for i in range(0,len(soups)): refs.append(str(soups[i])[74:79]) #print(refs) display = Display(visible=0, size=(800, 600)) display.start() browser = webdriver.Firefox() browser.get('https://www.crowd2fund.com/login') browser.find_element_by_name('_username').send_keys('martin@martinbrien.com') browser.find_element_by_name('_password').send_keys('02ebrienm') browser.find_element_by_name('_submit').click() #print(browser.title) for i in range(0,len(refs)): browser.get("https://www.crowd2fund.com/exchange/"+str(refs[i])+"/purchase") price = browser.find_element_by_name('trade[amount]').get_attribute("value") rate = browser.find_element_by_name('trade[interestRate]').get_attribute("value") company = browser.find_element_by_tag_name('h4').text term=browser.find_element_by_xpath("//div[@class='line']//span[1]").text print(refs[i]+","+price+","+rate) browser.quit() display.stop()