I am trying to scrape some prices from the website for different mattress sizes but the problem is that the dropdown menu is not of select type, it consists of a div with ul list of different options so when I try to click on an option it gives me this error. What can I do ?
here’s my code:
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.select import Select
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from bs4 import BeautifulSoup
from datetime import datetime
import time
import re
import logging
import traceback
from datetime import datetime, timedelta
import argparse
import pandas as pd
from sqlalchemy import create_engine
from sqlalchemy import exc
import mysql.connector as mysql
import io
import re
url ='https://thesleepcompany.in/product/mattress/smart-ortho/'
driver = webdriver.Chrome(options=options)
driver.get(url)
WebDriverWait(driver,15)
driver.find_element_by_css_selector('div.dk-selected.attached.enabled').click()
WebDriverWait(driver, 5)
opt= driver.find_element_by_css_selector('ul.dk-select-options').find_elements_by_tag_name('li')
driver.execute_script('arguments[0].setAttribute("class","dk-option attached enabled dk-option-selected")', opt[5])
driver.execute_script('arguments[0].setAttribute("aria-selected","true")', opt[5])
opt[5].click()
opt=driver.find_element_by_css_selector('ul.dk-select-options').find_elements_by_tag_name('li')
print_prices() #ignore it
print_price() #ignore it
this is the full error message I am getting:
ElementClickInterceptedException Traceback (most recent call last)
<ipython-input-69-94877ec06aa4> in <module>
4 driver.execute_script('arguments[0].setAttribute("class","dk-option attached enabled dk-option-selected")', opt[5])
5 driver.execute_script('arguments[0].setAttribute("aria-selected","true")', opt[5])
----> 6 opt[5].click()
7 opt=driver.find_element_by_css_selector('ul.dk-select-options').find_elements_by_tag_name('li')
8 print_prices()
~AppDataLocalProgramsPythonPython37libsite-packagesseleniumwebdriverremotewebelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
~AppDataLocalProgramsPythonPython37libsite-packagesseleniumwebdriverremotewebelement.py in _execute(self, command, params)
631 params = {}
632 params['id'] = self._id
--> 633 return self._parent.execute(command, params)
634
635 def find_element(self, by=By.ID, value=None):
~AppDataLocalProgramsPythonPython37libsite-packagesseleniumwebdriverremotewebdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~AppDataLocalProgramsPythonPython37libsite-packagesseleniumwebdriverremoteerrorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
ElementClickInterceptedException: Message: element click intercepted: Element <li class="dk-option attached enabled dk-option-selected" data-value="72%e2%80%b3-x-66%e2%80%b3-x-5%e2%80%b3" role="option" aria-selected="true" id="dk1-72%e2%80%b3-x-66%e2%80%b3-x-5%e2%80%b3">...</li> is not clickable at point (832, 483). Other element would receive the click: <div class="woocommerce-product-details__short-description">...</div>
(Session info: chrome=91.0.4472.101)
2
Answers
See if this works:-
We cannot use
Select
class since the drop down is not built usingSelect
andoption
tag. Instead it is Ul and li tags :Below code works for me :
Imports :