from pyvirtualdisplay import Display from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC import numpy import urllib.request from bs4 import BeautifulSoup import re high=4000 low=600 inc=40 #i=2 l = [i for i in range(1,int(((high-low)/inc)))] print(l) for i in l: gap="https://properties.movebubble.com/find-homes-to-rent?min-price="+str(low+(inc*i))+"&max-price="+str((low+(inc*i))+inc) print(gap) page=BeautifulSoup(urllib.request.urlopen(gap).read(), 'html.parser') totalItems=page.select_one("h1[class=total-results-text]").text totalItems=int(re.sub('[^0-9]','', totalItems)) print(totalItems)