2025, Dec 09 19:00
Selenium on Linux: Fix 'Too many open files' by closing leaks, reusing Xvfb, raising ulimit, and pooling WebDrivers
Stop Selenium jobs on Linux failing with 'Too many open files'. Close leaks, reuse Xvfb via pyvirtualdisplay, raise ulimit, and use a reusable WebDriver pool.
Long-running Selenium jobs on Linux sometimes die not with a crash in your logic, but with the operating system telling you it has had enough. If your parser runs smoothly for hours and then halts with a cryptic message, chances are you are leaking resources. The signal is unmistakable.
OSError: [Errno 24] Too many open files
This guide walks through the exact failure pattern, what triggers it in Selenium plus pyvirtualdisplay setups, and how to eliminate the leaks by closing everything deterministically, reusing the virtual display, bumping ulimit, and, where suitable, switching to a reusable WebDriver pool.
Problem setup
The scenario is a headless parser that rotates over search queries, launches a Chrome driver with proxy authentication, and uses pyvirtualdisplay on Linux. After 10 hours it crashes with “Too many open files.” Below is a representative version of the code that exhibits the issue. It keeps the same program structure and behavior while using different identifiers.
import logging
import time
import random
import platform
import traceback
import zipfile
from datetime import datetime
from selenium.webdriver.common.by import By
from fake_useragent import UserAgent
import undetected_chromedriver as uc
from selenium_stealth import stealth
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("scan_worker.log"),
logging.StreamHandler()
]
)
log = logging.getLogger('ScanLogger')
def pick_ua_chrome():
ua = UserAgent(browsers='chrome', os='windows', platforms='pc')
return ua.random
def make_driver(proxy_conf: dict, SHOW_UI=False):
chr_opts = uc.ChromeOptions()
if not SHOW_UI:
chr_opts.add_argument('--headless')
chr_opts.add_argument('--no-sandbox')
chr_opts.add_argument('--disable-dev-shm-usage')
chr_opts.add_argument('--start-maximized')
chr_opts.add_argument('--disable-blink-features=AutomationControlled')
chr_opts.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36")
manifest_json = """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"<all_urls>",
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
},
"minimum_chrome_version":"22.0.0"
}
"""
background_js = """
var config = {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "http",
host: "%s",
port: parseInt(%s)
},
bypassList: ["localhost"]
}
};
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
function callbackFn(details) {
return {
authCredentials: {
username: "%s",
password: "%s"
}
};
}
chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["<all_urls>"]},
['blocking']
);
""" % (proxy_conf["proxy"], proxy_conf["proxy_port"], proxy_conf["proxy_login"], proxy_conf["proxy_password"])
if proxy_conf:
ext_zip = 'proxy_auth_plugin.zip'
with zipfile.ZipFile(ext_zip, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
chr_opts.add_extension(ext_zip)
browser = uc.Chrome(chrome_options=chr_opts)
stealth(browser,
languages=["ru-RU", "ru", "en-US", "en"],
vendor="Google Inc.",
platform="Linux x86_64",
webgl_vendor="Intel Inc.",
renderer="Intel Open Source Technology Center Mesa DRI Intel(R) UHD Graphics 620 (KBL GT2)",
fix_hairline=True,
)
return browser
class YaSearchScraper:
def __init__(self, proxy_conf, SHOW_UI=False, SAVE_SHOT=False):
print(f"SHOW_UI={SHOW_UI} SAVE_SHOT={SAVE_SHOT}")
self.save_snap = SAVE_SHOT
self.proxy_conf = proxy_conf
self.show_ui = SHOW_UI
self.browser = None
self.vdisplay = None
self._boot_driver()
def _boot_driver(self):
if platform.system() == 'Darwin':
self._chr_opts = uc.ChromeOptions()
self._chr_opts.add_argument('--headless=new')
self._chr_opts.add_argument('--disable-gpu')
self._chr_opts.add_argument('--no-sandbox')
self._chr_opts.add_argument('--disable-dev-shm-usage')
self.browser = make_driver(proxy_conf=self.proxy_conf, SHOW_UI=self.show_ui)
else:
from pyvirtualdisplay import Display
self.vdisplay = Display(visible=0, size=(800, 600))
self.vdisplay.start()
self.browser = make_driver(proxy_conf=self.proxy_conf, SHOW_UI=self.show_ui)
self.browser.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
'source': '''
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
'''
})
self.browser.get("https://ya.ru/")
time.sleep(random.uniform(1.5, 3.5))
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.shutdown()
def shutdown(self):
try:
if self.browser:
self.browser.quit()
self.browser = None
except Exception as e:
log.error(f"Error closing driver: {e}")
try:
if self.vdisplay:
self.vdisplay.stop()
self.vdisplay = None
except Exception as e:
log.error(f"Error stopping display: {e}")
def tick_captcha(self):
now_str = str(datetime.now()).replace(' ', '_')
if "showcaptcha" in self.browser.current_url:
log.info("Captcha found")
if self.save_snap:
self.browser.save_screenshot(f'screens/img_captcha_{now_str}.png')
btns = self.browser.find_elements(By.XPATH, "//input[@class='CheckboxCaptcha-Button']")
if btns:
btns[0].click()
log.info("Button clicked")
time.sleep(15)
if self.save_snap:
self.browser.save_screenshot(f'screens/img_captcha_afterclick_{now_str}.png')
elif self.save_snap:
self.browser.save_screenshot(f'screens/img_{now_str}.png')
def harvest(self, title: str):
log.info(f"Start parse {title}")
found = []
try:
self.browser.get(f"https://ya.ru/search/?text={title}&lr=213&search_source=yaru_desktop_common&search_domain=yaru")
self.tick_captcha()
for i in range(1, 5):
found.extend(self.harvest_page(page_id=i))
self.flip_page()
self.tick_captcha()
time.sleep(random.uniform(2, 5))
except Exception:
log.error(f"Exception in {traceback.format_exc()}")
finally:
log.info(f"Found {len(found)} for film {title}: {found}")
def harvest_page(self, page_id):
acc = []
nodes = self.browser.find_elements(By.XPATH, value='//a[@class="Link Link_theme_normal OrganicTitle-Link organic__url link"]')
for node in nodes:
href = node.get_attribute("href")
if href and "yabs.yandex.ru" not in href:
acc.append(href)
log.info(f"Found {len(acc)} urls on page {page_id}")
return acc
def flip_page(self):
nxt = self.browser.find_elements(By.XPATH, '//div[@class="Pager-ListItem Pager-ListItem_type_next"]')
if nxt:
nxt[0].click()
time.sleep(random.uniform(3, 6))
if __name__ == "__main__":
proxy_book = [
# Proxy data
]
titles = ["Terminator smotret", "Sasha Tanya smotret", "John Wick smotret onlayn"]
turn = 0
while True:
try:
with YaSearchScraper(SHOW_UI=False, proxy_conf=proxy_book[turn % 2]) as runner:
item = titles[turn]
turn = (turn + 1) % len(titles)
runner.harvest(item)
time.sleep(random.uniform(8, 15))
except Exception:
log.error(f"Exception {traceback.format_exc()}")
time.sleep(1)
Why the file descriptors explode
The failure is a textbook case of resource accumulation during long runs. There are three pressure points. New virtual displays are created repeatedly with pyvirtualdisplay.Display and are not reliably torn down, so the process keeps file descriptors around. Selenium and Chrome naturally keep many descriptors open during work, so even small leaks add up. Linux enforces a default per-process open file limit, commonly 1024, and when the loop runs for hours, that limit is crossed.
How to stop the leak and keep the job healthy
The remediation comes in layers. First make shutdown bulletproof so that every iteration cleans up drivers, Xvfb instances, and temporary artifacts like the proxy extension. Then, reduce churn by reusing the display instead of creating it on each pass. Next, raise OS limits to a reasonable level for this workload. Add explicit garbage collection and small hygiene tweaks in Chrome options, and add periodic cooldowns to let the system breathe. Finally, consider a driver pool to eliminate constant driver creation entirely.
Harden the shutdown path
Ensure all resources are closed, and delete the proxy extension file if it was created. Below is the adjusted method that performs an ordered teardown and removes the zip.
def shutdown(self):
if self.browser:
try:
self.browser.quit()
except Exception as e:
log.error(f"Error closing driver: {e}")
finally:
self.browser = None
if self.vdisplay:
try:
self.vdisplay.stop()
except Exception as e:
log.error(f"Error stopping display: {e}")
finally:
self.vdisplay = None
import os
try:
ext_zip = 'proxy_auth_plugin.zip'
if os.path.exists(ext_zip):
os.remove(ext_zip)
except Exception as e:
log.error(f"Error removing plugin file: {e}")
Reuse the virtual display once
Creating a new Xvfb display in every cycle is wasteful. A class-level singleton display prevents per-iteration spawning. The display is started once and shared by all instances.
class YaSearchScraper:
_shared_display = None
def _boot_driver(self):
if platform.system() == 'Darwin':
self._chr_opts = uc.ChromeOptions()
self._chr_opts.add_argument('--headless=new')
self._chr_opts.add_argument('--disable-gpu')
self._chr_opts.add_argument('--no-sandbox')
self._chr_opts.add_argument('--disable-dev-shm-usage')
self.browser = make_driver(proxy_conf=self.proxy_conf, SHOW_UI=self.show_ui)
else:
from pyvirtualdisplay import Display
if YaSearchScraper._shared_display is None:
YaSearchScraper._shared_display = Display(visible=0, size=(800, 600))
YaSearchScraper._shared_display.start()
self.vdisplay = YaSearchScraper._shared_display
self.browser = make_driver(proxy_conf=self.proxy_conf, SHOW_UI=self.show_ui)
Raise the OS descriptor limits
When the workload is legitimately heavy, lifting limits makes sense. Edit the limits configuration and set nofile higher. Use elevated permissions if needed.
nano /etc/security/limits.conf
Add the lines below and save with Ctrl + x, then y, then Enter.
* soft nofile 4096
* hard nofile 8192
Add explicit GC and Chrome lightening flags
Triggering garbage collection after each iteration helps reclaim Python-level objects sooner. This aligns well with deterministic driver shutdown. Additional Chrome flags can trim resource usage.
import gc
# in the loop's finally or maintenance section
gc.collect()
# when assembling options for Chrome
chr_opts.add_argument('--disable-extensions')
chr_opts.add_argument('--disable-software-rasterizer')
chr_opts.add_argument('--disable-logging')
Pace the loop to relieve pressure
A strategic pause every few iterations gives the system leeway to finish background cleanup.
if turn % 10 == 0:
time.sleep(30)
Make the main loop resilient
Putting the pieces together, the loop can pace itself, collect garbage, and stop the shared display on program shutdown.
if __name__ == "__main__":
proxy_book = [/* your proxies */]
titles = ["Terminator smotret", "Sasha Tanya smotret", "John Wick smotret onlayn"]
turn = 0
try:
while True:
try:
with YaSearchScraper(SHOW_UI=False, proxy_conf=proxy_book[turn % 2]) as runner:
item = titles[turn]
turn = (turn + 1) % len(titles)
runner.harvest(item)
if turn % 10 == 0:
time.sleep(30)
import gc
gc.collect()
else:
time.sleep(random.uniform(8, 15))
except Exception:
log.error(f"Exception {traceback.format_exc()}")
time.sleep(30)
finally:
if YaSearchScraper._shared_display:
YaSearchScraper._shared_display.stop()
Pool WebDriver instances instead of recreating them
Another effective mitigation is to avoid spinning up a new driver per request. A small pool, reused across tasks, significantly reduces file descriptor churn and startup overhead. The implementation below builds a pool of Chrome drivers with proxy support and stealth tweaks, returns them to the pool after use, and self-heals broken drivers.
from selenium.webdriver import Chrome
from selenium.webdriver.chrome.options import Options
import threading
import queue
import time
import random
import logging
from fake_useragent import UserAgent
import zipfile
import os
from selenium_stealth import stealth
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler("driver_pool.log"),
logging.StreamHandler()
]
)
pool_log = logging.getLogger('DriverPoolLogger')
class DriverStash:
def __init__(self, pool_size=3, proxy_conf=None, headless=True):
self.pool_size = pool_size
self.proxy_conf = proxy_conf
self.headless = headless
self._q = queue.Queue(maxsize=pool_size)
self._lock = threading.Lock()
self._seed()
def _spawn(self):
opts = Options()
if self.headless:
opts.add_argument('--headless')
opts.add_argument('--no-sandbox')
opts.add_argument('--disable-dev-shm-usage')
opts.add_argument('--start-maximized')
opts.add_argument('--disable-blink-features=AutomationControlled')
ua = UserAgent(browsers='chrome', os='windows', platforms='pc')
opts.add_argument(f"user-agent={ua.random}")
if self.proxy_conf:
addon = self._proxy_addon()
opts.add_extension(addon)
d = Chrome(options=opts)
stealth(d,
languages=["ru-RU", "ru", "en-US", "en"],
vendor="Google Inc.",
platform="Linux x86_64",
webgl_vendor="Intel Inc.",
renderer="Intel Open Source Technology Center Mesa DRI Intel(R) UHD Graphics 620 (KBL GT2)",
fix_hairline=True,
)
d.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {
'source': '''
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
'''
})
return d
def _proxy_addon(self):
manifest_json = """
{
"version": "1.0.0",
"manifest_version": 2,
"name": "Chrome Proxy",
"permissions": [
"proxy",
"tabs",
"unlimitedStorage",
"storage",
"<all_urls>",
"webRequest",
"webRequestBlocking"
],
"background": {
"scripts": ["background.js"]
},
"minimum_chrome_version":"22.0.0"
}
"""
background_js = """
var config = {
mode: "fixed_servers",
rules: {
singleProxy: {
scheme: "http",
host: "%s",
port: parseInt(%s)
},
bypassList: ["localhost"]
}
};
chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});
function callbackFn(details) {
return {
authCredentials: {
username: "%s",
password: "%s"
}
};
}
chrome.webRequest.onAuthRequired.addListener(
callbackFn,
{urls: ["<all_urls>"]},
['blocking']
);
""" % (self.proxy_conf["proxy"], self.proxy_conf["proxy_port"],
self.proxy_conf["proxy_login"], self.proxy_conf["proxy_password"])
addon_zip = 'proxy_auth_plugin.zip'
with zipfile.ZipFile(addon_zip, 'w') as zp:
zp.writestr("manifest.json", manifest_json)
zp.writestr("background.js", background_js)
return addon_zip
def _seed(self):
for _ in range(self.pool_size):
d = self._spawn()
self._q.put(d)
def acquire(self, timeout=30):
try:
d = self._q.get(timeout=timeout)
d.delete_all_cookies()
return d
except queue.Empty:
raise TimeoutError("No available drivers in the pool")
def release(self, d):
try:
d.delete_all_cookies()
d.get("about:blank")
self._q.put(d)
except Exception as e:
pool_log.error(f"Error returning driver to pool: {e}")
try:
d.quit()
except Exception:
pass
fresh = self._spawn()
self._q.put(fresh)
def cleanup(self):
while not self._q.empty():
try:
d = self._q.get_nowait()
d.quit()
except Exception as e:
pool_log.error(f"Error cleaning up driver: {e}")
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.cleanup()
An updated program skeleton using two proxy-backed pools follows. It rotates pools and periodically performs maintenance sleeps to keep resource use steady.
if __name__ == "__main__":
proxies = [
{"proxy": "proxy1.example.com", "proxy_port": "8080", "proxy_login": "user1", "proxy_password": "pass1"},
{"proxy": "proxy2.example.com", "proxy_port": "8080", "proxy_login": "user2", "proxy_password": "pass2"}
]
titles = ["Terminator smotret", "Sasha Tanya smotret", "John Wick smotret onlayn"]
farms = [
DriverStash(pool_size=2, proxy_conf=proxies[0], headless=True),
DriverStash(pool_size=2, proxy_conf=proxies[1], headless=True)
]
k = 0
rounds = 0
try:
while True:
try:
farm_idx = k % len(farms)
title = titles[k % len(titles)]
# Example usage if YaSearchScraper integrates with a driver pool
# with YaSearchScraper(driver_pool=farms[farm_idx]) as runner:
# runner.harvest(title)
k += 1
rounds += 1
if rounds % 20 == 0:
pool_log.info("Performing periodic maintenance")
time.sleep(30)
time.sleep(random.uniform(8, 15))
except Exception:
pool_log.error(f"Exception {traceback.format_exc()}")
time.sleep(30)
finally:
for farm in farms:
farm.cleanup()
Why this knowledge matters
Long-running scraping or automation systems seldom fail because of a single catastrophic event. They degrade through tiny leaks that the OS eventually refuses to tolerate. Understanding how file descriptors accumulate, where Selenium and Xvfb consume them, and how ulimit gates your process gives you the levers to build stable services. This is especially important for headless Linux workloads that cycle drivers, proxies, and stealth settings for days.
Practical closing notes
A few operational habits go a long way. Monitor open file counts and the process’s soft and hard limits with standard tools, and keep an eye on lsof output during extended runs to confirm that descriptors are not monotonically increasing. If a single-display approach still shows growth, revisit the shutdown order and verify that temporary artifacts are actually removed. Reuse where possible, from displays to drivers, and slow the loop periodically to allow cleanup to catch up.
The fixes above remain simple: shut down aggressively, reuse the Display, raise the nofile ceiling sensibly, collect garbage, lighten Chrome’s footprint, and, where appropriate, stop creating drivers in a hot loop by pooling them. With these in place, the “Too many open files” wall stops being an inevitability and becomes a solved operational concern.