skip to Main Content

I am trying to close browser even if the script is not exited normally. I have tried few methods which I found on internet but failed to do so. First was to use try/except and finally. I implemented it but if does not works if the script execution is forcefully stopped.
I have reproduced the code I have. It is not closing the browser on force exit if using chrome. Firefox is closing without any problem. The code is divided in two files (scraper.py, initialize.py)

scraper.py

try:
    from .initialize import Init
except:
    from initialize import Init
import time

URL = "https://web.whatsapp.com/"
def main(driver):
     # do parsing and other stuff
     driver.get(URL)
     time.sleep(15)

def get_driver(browser, headless):
    # calling start drivers method of init class from initialize file
    return Init(method='profile',write_directory='output/',selenium_wire=True,undetected_chromedriver=True,old_profile_using=False).start_driver(browser=browser, url=URL, headless=headless, refresh=True)
 

try:
    driver = get_driver("firefox" , False)
    main(driver)
    # get_driver returns driver.
except Exception as e:
    print(f"Error: {e}")
finally:
        try:
            driver.quit()
        except NameError:
            pass

initialize.py

from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
import shutil, os, sys, getpass, browser_cookie3, json, traceback
from time import sleep
from webdriver_manager.firefox import GeckoDriverManager
from webdriver_manager import utils

"""def drivers():
    driver_installation = ChromeDriverManager().install()
    service = Service(driver_installation)
    driver = webdriver.Chrome(service=service)
    return driver

class Init:
    def __init__(self, browser):
        self.browser = browser
    def get_driver(self): #for sake of simplicity this is the get_driver function
        driver = drivers()
        return driver"""

class Init:
    basename = os.path.basename(sys.argv[0]).split(".")[0]  
    homedir  = os.path.join("C:\ProgramData", basename, getpass.getuser())
    #WRITE_DIRECTORY = 'output_files'
    #WRITE_DIRECTORY = f'{homedir}output_files/'
    WRITE_DIRECTORY = os.path.join(homedir,'output_files')

    def __init__(self, method, write_directory=WRITE_DIRECTORY, selenium_wire=False,
                 undetected_chromedriver=False, old_profile_using=False):
        self.method = method
        self.write_directory = write_directory
        self.selenium_wire = selenium_wire
        self.undetected_chromedriver = undetected_chromedriver
        self.old_profile_using = old_profile_using

        if not any((self.selenium_wire, self.undetected_chromedriver)):
            raise Exception("Specify at least one driver")
        if method not in ['profile', 'cookies']:
            raise Exception(f"Unexpected method {method=}")

    @staticmethod
    def find_path_to_browser_profile(browser):
        """Getting path to firefox or chrome default profile"""
        if sys.platform == 'win32':
            browser_path = {
                'chrome': os.path.join(
                    os.environ.get('LOCALAPPDATA'), 'Google', 'Chrome', 'User Data'),
                "firefox": os.path.join(
                    os.environ.get('APPDATA'), 'Mozilla', 'Firefox')
            }
        else:
            browser_path = {
                'chrome': os.path.expanduser('~/.config/google-chrome'),
                "firefox": os.path.expanduser('~/.mozilla/firefox')
            }
        path_to_profiles = browser_path[browser]

        print(f'{path_to_profiles=}')
        if os.path.exists(path_to_profiles):
            if browser == 'firefox':
                return browser_cookie3.Firefox.get_default_profile(path_to_profiles)
            if browser == 'chrome':
                return path_to_profiles
        else:
            print(f"{browser} profile - not found")

    def _remove_old_db(self, path_to_browser_profile, browser):

        if browser == 'firefox':
            path_to_browser_profile = os.path.join(path_to_browser_profile, 'storage', 'default')
            folder_list = [i for i in
                           os.listdir(path_to_browser_profile) if 'to-do.live' in i]
            if folder_list:
                path_to_folder = os.path.join(path_to_browser_profile, folder_list[0], 'idb')
                try:
                    shutil.rmtree(path_to_folder)
                    print('removed')
                    return
                except:
                    sleep(2)
                    pass

        elif browser == 'chrome':
            path_to_browser_profile = os.path.join(path_to_browser_profile, 'Default', 'IndexedDB')
        else:
            raise Exception('browser not supported')
        print(path_to_browser_profile)
        folder_list = [i for i in
                       os.listdir(path_to_browser_profile) if 'to-do.live' in i]
        if folder_list:
            print(folder_list[0])
            path_to_folder = os.path.join(path_to_browser_profile, folder_list[0])
            shutil.rmtree(path_to_folder)
            print('removed')

    def init_driver(self, browser, path_to_profile=None, headless=False, remove_old_session=False):
        """Initialize webdriver"""
        print(f"Initialize webdriver for {browser}")
        if self.selenium_wire and self.undetected_chromedriver:
            from seleniumwire.undetected_chromedriver.v2 import Chrome, ChromeOptions # working
        elif self.selenium_wire:
            from seleniumwire.webdriver import Chrome, ChromeOptions # not working
        else:
            from undetected_chromedriver import Chrome, ChromeOptions

        if self.method == 'profile':
            path_to_profile = self.copy_profile(browser)
            if remove_old_session:
                self._remove_old_db(path_to_profile, browser)

        if 'firefox' in browser:
            wdm_path = os.path.join(os.path.expanduser("~"), ".wdm", "drivers.json")
            print(wdm_path)
            with open(wdm_path, "r") as f:
                driver_logs = json.load(f)
            gecko_drivers = {}
            g_count = 0
            for drivers, val in driver_logs.items():
                if "geckodriver" in drivers:
                    g_count += 1
                    gecko_drivers[drivers] = val
            firefox_options = webdriver.FirefoxOptions()
            # firefox_options.add_argument('--no-sandbox')
            if self.method == 'profile':
                path_to_firefox_profile = path_to_profile
                profile = webdriver.FirefoxProfile(path_to_firefox_profile)
                profile.accept_untrusted_certs = True
                profile.set_preference("dom.webdriver.enabled", False)
                profile.set_preference('useAutomationExtension', False)
                profile.set_preference("security.mixed_content.block_active_content", False)
                profile.update_preferences()
                firefox_options.set_preference("general.useragent.override", 'user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0')
                #firefox_options.add_argument(
                #    'user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0')
                if headless:
                    firefox_options.add_argument("--headless")
                firefox_options.add_argument('--ignore-certificate-errors')
                firefox_options.add_argument("--width=1400")
                firefox_options.add_argument("--height=1000")
                try:
                    driver_installation = GeckoDriverManager().install()
                except Exception as e:
                    c = 1
                    if "rate limit" in str(e):
                        for i in gecko_drivers.values():
                            if c==g_count:
                                driver_installation = i["binary_path"]
                                break
                            c +=1
                service = Service(driver_installation)
                if sys.platform == 'win32':
                    from subprocess import CREATE_NO_WINDOW
                    service.creationflags = CREATE_NO_WINDOW

                driver = webdriver.Firefox(options=firefox_options, firefox_profile=profile,
                                           service=service)
                driver.implicitly_wait(10)
            else:
                firefox_options = webdriver.FirefoxOptions()
                firefox_options.add_argument('--incognito')
                firefox_options.add_argument("--disable-blink-features=AutomationControlled")
                firefox_options.add_argument(
                    'user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0')
                firefox_options.add_argument("--mute-audio")
                if headless:
                    firefox_options.add_argument("--headless")
                try:
                    driver_installation = GeckoDriverManager().install()
                except Exception as e:
                    if "rate limit" in str(e):
                        c = 1
                        for i in gecko_drivers.values():
                            if c==g_count:
                                driver_installation = i["binary_path"]
                            c +=1

                service = Service(driver_installation)
                if sys.platform == 'win32':
                    from subprocess import CREATE_NO_WINDOW
                    service.creationflags = CREATE_NO_WINDOW
                driver = webdriver.Firefox(options=firefox_options, executable_path=driver_installation,
                                           service=service)
            return driver
        elif 'chrome' in browser:
            chrome_options = ChromeOptions()
            if self.method == 'profile':
                chrome_options.add_argument('--no-first-run --no-service-autorun --password-store=basic')
                chrome_options.add_argument("--disable-extensions")
                # chrome_options.user_data_dir = path_to_profile
                chrome_options.add_argument("--window-size=1400,1000")
                chrome_options.add_argument(
                    'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36')
                if sys.platform == 'linux':
                    chrome_options.binary_location = '/bin/google-chrome'
                if headless:
                    chrome_options.add_argument("--headless")
                # You cannot use default chromedriver for google services, because it was made by google.
                browser_version = int(utils.get_browser_version_from_os('google-chrome').split('.')[0])
                print(f"{browser_version=}")
                driver = Chrome(options=chrome_options, version_main=browser_version, patcher_force_close=True,
                                user_data_dir=path_to_profile)
                driver.implicitly_wait(10)
                return driver
            else:
                chrome_options.add_argument('--incognito')
                chrome_options.add_argument("--disable-blink-features=AutomationControlled")
                chrome_options.add_argument('--no-sandbox')
                chrome_options.add_argument("--mute-audio")
                chrome_options.add_argument("--window-size=1400,1000")
                if headless:
                    chrome_options.add_argument("--headless")
                chrome_options.add_argument(
                    'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36')
                driver_installation = ChromeDriverManager().install()
                service = Service(driver_installation)
                if sys.platform == 'win32':
                    from subprocess import CREATE_NO_WINDOW
                    service.creationflags = CREATE_NO_WINDOW
                driver = webdriver.Chrome(driver_installation, options=chrome_options, service=service)
            return driver

    def copy_profile(self, browser):
        """Copy browsers profile to /home/{username}/browser_profiles/{browser} directory and returns path to it"""
        path_to_profile = self.find_path_to_browser_profile(browser)
        sys_path_to_copy = os.path.join(os.getcwd(), self.write_directory, browser)
        print(f'copy_profile() initializer: sys_path_to_copy: {sys_path_to_copy}')

        if browser == 'chrome':
            if self.old_profile_using:
                print('Using old profile')
                return sys_path_to_copy
            if not os.path.exists(sys_path_to_copy):
                try:
                    shutil.copytree(path_to_profile, sys_path_to_copy,
                                    symlinks=True, ignore_dangling_symlinks=True, dirs_exist_ok=True)
                except:
                    pass
                print(f'{browser} profile copied from {path_to_profile} to  {sys_path_to_copy}')
            if sys.platform == 'win32':
                 try:  
                    shutil.copytree(os.path.join(sys_path_to_copy,'Default','Network'), os.path.join(sys_path_to_copy,'Default'),dirs_exist_ok=True)
                 except:
                    pass
            return sys_path_to_copy
        elif browser == 'firefox':
            firefox_dir = path_to_profile.split('/')[-1]
            if self.old_profile_using:
                print('Using old profile')
                return os.path.join(sys_path_to_copy, firefox_dir)
            if not os.path.exists(sys_path_to_copy):
                try:
                    shutil.copytree(path_to_profile, os.path.join(sys_path_to_copy, firefox_dir),
                                    ignore_dangling_symlinks=True)
                except:
                    pass
                print(
                    f'{browser} profile copied from {path_to_profile} to  {sys_path_to_copy}/{firefox_dir}')
            return os.path.join(sys_path_to_copy, firefox_dir)


    def start_driver(self, url, browser, headless, refresh=True, remove_old_session=False):
        """Prepering folders and init webdriver"""

        self.create_folders(self.write_directory)
        if self.method == "profile":
            path_to_profile = f'{self.write_directory}{browser}'
            if not self.old_profile_using:
                self.remove_profile_folder(path_to_profile)

        driver = self.init_driver(browser, headless=headless, remove_old_session=remove_old_session)
        try:
            sleep(3)
            driver.get(url)
            sleep(5)
            if refresh and "telegram" not in url:
                driver.refresh()
            print(driver.current_url)
            if driver.current_url == "about:blank":
                driver.get(url)
            sleep(5)
            return driver
        except:
            # raise Exception
            print(traceback.format_exc())
            driver.close()
            driver.quit()

    def remove_profile_folder(self, path_to_profile_folder):
        """Removes browser profile folder"""
        print("Removing old profile")
        while os.path.exists(path_to_profile_folder):
            try:
                shutil.rmtree(path_to_profile_folder)
            except Exception as ex:
                print(ex)
                sleep(2)
                pass

    def create_folders(self, folder_path):
        if not os.path.exists(folder_path):
            print(f'initializer() Creating folder for {folder_path}')
            os.makedirs(folder_path)  # Creating folder
        return folder_path

Things I tried

try:
    driver = get_driver(browser)
    # get_driver returns driver.
    main(
        driver,
    )
except:
    print(traceback.format_exc())
finally:
        try:
            driver.quit()
        except NameError:
            pass

The other thing I tried was to use atexit

import atexit
def get_driver(browser): #for sake of simplicity this is the get_driver function
    driver_installation = ChromeDriverManager().install()
    service = Service(driver_installation)
    driver = webdriver.Chrome(service=service)
    return driver

@atexit.register()
def cleanup():
     driver.close()
     driver.quit()

try:
    driver = get_driver(browser)
    # get_driver returns driver.
    main(
        driver,
    )
except:
    print(traceback.format_exc())
finally:
        try:
            driver.quit()
        except NameError:
            pass

Note, I tries the above in my main code not like this. These also work fine in simple script but not in the code I have provided above.
I do not know if I am using it correctly or not.
The browser is closed on force exit if I am executing the script using VS Code debugger but it fails in simple execution.
Thanks in advance for any help.

2

Answers


  1. If you want to close & destroy the WebDriver instance and the Web Client instances gracefully all you need is:

    driver.quit()
    

    You don’t need to invoke driver.close() before driver.quit()

    Login or Signup to reply.
  2. You are missing a None check before attempting to close the driver. Also, driver.close() closes a single browser tab and driver.quit() closes the browser along with all open tabs. driver.quit() alone should take care of your situation.

    finally:
        if driver:
            driver.quit()
    
    Login or Signup to reply.
Please signup or login to give your own answer.
Back To Top
Search