Task 081221 Selenium

december/task_011221/task_011221_ss_scraper.py

@@ -18,24 +18,14 @@ class SS:
 		self.url = url
 
 	def _get_page_amount(self):
-		current_page = None
+		page = requests.get(self.url, headers=HEADERS)
-		page_amount = 1
+		soup = BeautifulSoup(page.content, 'html.parser')
-		url = self.url
-		while current_page != page_amount:
-			current_page = page_amount
-			page = requests.get(url, headers=HEADERS)
-			soup = BeautifulSoup(page.content, 'html.parser')
 
-			# getting max page amount
+		last_url = soup.find(class_='td2').findChild('a')['href']
-			for el in soup.find_all(class_='navi'):
+		page_amount = last_url[last_url.find("page") + 4:last_url.find(".html")]
-				cropped_number = el.get_text().strip()
-				if cropped_number.isnumeric():
-					cropped_number = int(cropped_number)
-					if cropped_number > page_amount:
-						page_amount = cropped_number
-			url = self.url + f"/page{page_amount}.html"
 		print(f"Page amount = {page_amount}")
-		return page_amount
+
+		return int(page_amount)
 
 	def get_data(self):
 		items = []
@@ -63,7 +53,7 @@ class SS:
 				response = requests.get(image_url)
 				img = Image.open(BytesIO(response.content))
 				images.append(img)
-
+				print(img)
 				for elem in el.find_all(class_='msga2-o pp6'):
 					items.append(elem.get_text())
 
@@ -115,6 +105,7 @@ class SS:
 
 
 flats = SS("https://www.ss.com/lv/real-estate/flats/riga/all/sell/")
+flats2 = SS("https://www.ss.com/lv/real-estate/flats/riga-region/all/sell/")
 
 
 def main():

december/task_081221/task_081221.py

@@ -0,0 +1,44 @@
+# Date - 08.12.2021
+# Author - Kristiāns Francis Cagulis
+# Title - Class work 081221 Selenium
+
+import time
+from selenium import webdriver
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.common.by import By
+
+firefox = True
+
+if firefox:
+	browser = webdriver.Firefox()
+else:
+	browser = webdriver.Chrome("chromedriver")
+
+address = "https://www.riga.lv/lv"
+browser.get(address)
+time.sleep(3)
+agreement = browser.find_element_by_class_name('cookie-accept-all')
+agreement.click()
+
+search = browser.find_element_by_class_name('search-link')
+search.click()
+
+delay = 2
+WebDriverWait(browser, delay).until(EC.presence_of_all_elements_located((By.ID, 'edit-search')))
+search = browser.find_element_by_id('edit-search')
+search.send_keys("dokum")  # writes in search line
+
+search = browser.find_element_by_id('search-header-button')
+search.click()
+
+browser.maximize_window()
+WebDriverWait(browser, delay).until(EC.presence_of_all_elements_located((By.CLASS_NAME, 'filter-content')))
+delay = 3
+WebDriverWait(browser, delay).until(EC.presence_of_all_elements_located((By.ID, 'filter_type_content')))
+
+filter = browser.find_element_by_css_selector("label[for='filter_type_file']")
+filter.click()
+
+search = browser.find_element_by_id('search-view-button')
+search.click()