I am trying to make use of Thumbnail caching in Apache Superset. But, on a fresh install (from scratch) on a compute Engine, superset crashes when the first Thumbnail needs to be created. (I was able to connect to my database and create a first Chart, without a problem, but then when the thumbnail for this chart supposed to be create, superset crashes.)
Installed services, configs and logs below, I would appreciate any help/indication.
I installed:
- firefox-esr and GECKODRIVER_VERSION=0.30.0 using:
wget --no-verbose -O $relevant_path/firefox/geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/v$GECKODRIVER_VERSION/geckodriver-v$GECKODRIVER_VERSION-linux64.tar.gz \
&& sudo rm -rf /opt/geckodriver \
&& sudo tar -C /opt -zxf $relevant_path/firefox/geckodriver.tar.gz \
&& sudo rm $relevant_path/firefox/geckodriver.tar.gz \
&& sudo mv /opt/geckodriver /opt/geckodriver-$GECKODRIVER_VERSION \
&& sudo chmod 755 /opt/geckodriver-$GECKODRIVER_VERSION \
&& sudo ln -fs /opt/geckodriver-$GECKODRIVER_VERSION /usr/bin/geckodriver \
&& sudo ln -fs /opt/geckodriver-$GECKODRIVER_VERSION /usr/bin/wires
- redis as a backend. It is up and running on the same machine. Chart Data and others are successfully cached.
- celery initiated on the same machine as follows:
celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4
celery --app=superset.tasks.celery_app:app beat
Superset itself starts-up using:
superset init
gunicorn -w 5 \
-k gevent \
--timeout 120 \
-b 0.0.0.0:8088 \
--limit-request-line 0 \
--limit-request-field_size 0 \
--statsd-host localhost:8125 \
"superset.app:create_app()"
My full superset_config is:
############################## Imports ##############################
from celery.schedules import crontab
from superset.typing import CacheConfig
from typing import Any, Callable, Dict, List, Optional, Type, TYPE_CHECKING, Union
import logging
import os
import pkg_resources
from cachelib import RedisCache
############################## General Settings ##############################
SUPERSET_WEBSERVER_PORT = 8088
WTF_CSRF_ENABLED = True
SECRET_KEY = 'mykey'
############################## Enable Features ##############################
# some features are deactivated by defaut. We need the following
FEATURE_FLAGS = {
"THUMBNAILS": True,
"THUMBNAILS_SQLA_LISTENERS": True,
"DASHBOARD_CACHE": True,
"ALERT_REPORTS": True,
"UX_BETA": True,
"DASHBOARD_RBAC": True,
"LISTVIEWS_DEFAULT_CARD_VIEW" : True
}
############################## Logging uses Statsd ##############################
# uses Statsd as Logging engine (needs to be installed and run on the machine)
#default: LOG_LEVEL = DEBUG
BASE_DIR = pkg_resources.resource_filename("superset", "")
if "SUPERSET_HOME" in os.environ:
DATA_DIR = os.environ["SUPERSET_HOME"]
else:
DATA_DIR = os.path.expanduser("~/.superset")
LOG_LEVEL = "DEBUG"
#filename location needs to be changed
ENABLE_TIME_ROTATE = True
TIME_ROTATE_LOG_LEVEL = "DEBUG"
FILENAME = "/superset/superset.log"
ROLLOVER = "midnight"
INTERVAL = 1
BACKUP_COUNT = 14
############################## Language ##############################
BABEL_DEFAULT_LOCALE = "de"
BABEL_DEFAULT_FOLDER = "superset/translations"
LANGUAGES = {
'en': {'flag': 'us', 'name': 'English'},
'de': {'flag': 'de', 'name': 'German'}
}
############################## Meta Data management ##############################
# save meta data in MySQL (MariaDB is not supported yet)
# pip install pymysql is needed
SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://user:pass@localhost:3306/Superset_Metadata'
############################## Caching in Superset ##############################
# Data and Metadata Caching uses a REDIS in memory server (needs to be installed and run on the machine)
REDIS_HOST = "127.0.0.1"
REDIS_PORT = "6379"
# Metadata Caching
CACHE_CONFIG: CacheConfig = {
'CACHE_TYPE': 'redis',
'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day
'CACHE_KEY_PREFIX': 'superset_',
'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
}
#Caching the Data from the Database shown in the Dashboards
DATA_CACHE_CONFIG: CacheConfig = {
'CACHE_TYPE': 'redis',
'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day
'CACHE_KEY_PREFIX': 'data_',
'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
}
# Cache for filters state
FILTER_STATE_CACHE_CONFIG: CacheConfig = {
"CACHE_TYPE": "redis",
'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day
'CACHE_KEY_PREFIX': 'filter_',
"CACHE_THRESHOLD": 0,
"REFRESH_TIMEOUT_ON_RETRIEVAL": True,
'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
}
# Cache for chart form data
EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = {
"CACHE_TYPE": "redis",
'CACHE_DEFAULT_TIMEOUT': 24*60*60, # 1 day
'CACHE_KEY_PREFIX': 'Chart_',
"CACHE_THRESHOLD": 0,
"REFRESH_TIMEOUT_ON_RETRIEVAL": True,
'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
}
# Caches the top 5 Dashboards at 6:30 am
CELERYBEAT_SCHEDULE = {
'cache-warmup-hourly': {
'task': 'cache-warmup',
'schedule': crontab(minute=30, hour='6'),
'kwargs': {
'strategy_name': 'top_n_dashboards',
'top_n': 5,
'since': '7 days ago',
},
},
}
############################## Alerts and Reports ##############################
# they need a browser installed. Default config uses firefox
# the celery worker (asynch workers) is needed too and needs to be up and running
class CeleryConfig:
BROKER_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
CELERY_IMPORTS = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", )
CELERY_RESULT_BACKEND = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
CELERYD_PREFETCH_MULTIPLIER = 10
CELERY_ACKS_LATE = True
CELERY_ANNOTATIONS = {
'sql_lab.get_sql_results': {
'rate_limit': '100/s',
},
'email_reports.send': {
'rate_limit': '1/s',
'time_limit': 120,
'soft_time_limit': 150,
'ignore_result': True,
},
}
CELERYBEAT_SCHEDULE = {
'email_reports.schedule_hourly': {
'task': 'email_reports.schedule_hourly',
'schedule': crontab(minute=1, hour='*'),
},
'reports.scheduler': {
'task': 'reports.scheduler',
'schedule': crontab(minute='*', hour='*'),
},
'reports.prune_log': {
'task': 'reports.prune_log',
'schedule': crontab(minute=0, hour=0),
},
}
CELERY_CONFIG = CeleryConfig
SCREENSHOT_LOCATE_WAIT = 100
SCREENSHOT_LOAD_WAIT = 600
# This is for internal use, you can keep http
WEBDRIVER_BASEURL="http://superset:8088"
# This is the link sent to the recipient, change to your domain eg. https://superset.mydomain.com
WEBDRIVER_BASEURL_USER_FRIENDLY="http://localhost:8088"
# these are the default settings and included for reference reasons
WEBDRIVER_TYPE= "firefox"
WEBDRIVER_OPTION_ARGS = [
"--force-device-scale-factor=2.0",
"--high-dpi-support=2.0",
"--headless",
"--disable-gpu",
"--disable-dev-shm-usage",
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-extensions"
]
# Thumbnail Caching
THUMBNAIL_SELENIUM_USER = "admin"
THUMBNAIL_CACHE_CONFIG: CacheConfig = {
'CACHE_TYPE': 'redis',
'CACHE_DEFAULT_TIMEOUT': 24*60*60*30,
'CACHE_KEY_PREFIX': 'thumbnail_',
'CACHE_NO_NULL_WARNING': True,
'CACHE_REDIS_URL': 'redis://%s:%s/1' % (REDIS_HOST, REDIS_PORT)
}
from superset.stats_logger import StatsdStatsLogger
STATS_LOGGER = StatsdStatsLogger(host='localhost', port=8125, prefix='superset')
Superset reports the following:
Cache key: dada8ca5577d88d6f8cee79dd3306162
2022-03-03 15:33:46,987:INFO:superset.viz:Cache key: dada8ca5577d88d6f8cee79dd3306162
Serving from cache
2022-03-03 15:33:47,165:INFO:superset.viz:Serving from cache
2022-03-03 15:33:51,603:DEBUG:parsedatetime:eval - with context - False, False
Cache key: dada8ca5577d88d6f8cee79dd3306162
2022-03-03 15:33:51,756:INFO:superset.viz:Cache key: dada8ca5577d88d6f8cee79dd3306162
Serving from cache
2022-03-03 15:33:51,898:INFO:superset.viz:Serving from cache
404 Not Found: The requested URL was not found on the server. If you entered the URL manually please check your spe
lling and try again.
2022-03-03 15:33:52,684:WARNING:superset.views.base:404 Not Found: The requested URL was not found on the server. I
f you entered the URL manually please check your spelling and try again.
Attempting to get from cache: ffbbeb81838037a8e94d62e36d05610c
2022-03-03 15:33:59,633:INFO:superset.utils.screenshots:Attempting to get from cache: ffbbeb81838037a8e94d62e36d056
10c
Failed at getting from cache: ffbbeb81838037a8e94d62e36d05610c
2022-03-03 15:33:59,719:INFO:superset.utils.screenshots:Failed at getting from cache: ffbbeb81838037a8e94d62e36d056
10c
Triggering thumbnail compute (chart id: 1) ASYNC
2022-03-03 15:33:59,719:INFO:superset.charts.api:Triggering thumbnail compute (chart id: 1) ASYNC
Celery brings up the following:
[2022-03-03 15:30:52,540: INFO/ForkPoolWorker-4] Scheduled email reports not enabled in config
Caching chart: http://superset:8088/superset/slice/1/?standalone=true
[2022-03-03 15:33:44,636: INFO/ForkPoolWorker-1] Caching chart: http://superset:8088/superset/slice/1/?standalone=t
rue
Processing url for thumbnail: ffbbeb81838037a8e94d62e36d05610c
[2022-03-03 15:33:44,727: INFO/ForkPoolWorker-1] Processing url for thumbnail: ffbbeb81838037a8e94d62e36d05610c
Init selenium driver
[2022-03-03 15:33:44,728: INFO/ForkPoolWorker-1] Init selenium driver
Caching chart: http://superset:8088/superset/slice/1/?standalone=true
[2022-03-03 15:34:01,243: INFO/ForkPoolWorker-2] Caching chart: http://superset:8088/superset/slice/1/?standalone=t
rue
Failed at generating thumbnail Message: Connection refused (os error 111)
Traceback (most recent call last):
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/screenshots
.py", line 152, in compute_and_cache
payload = self.get_screenshot(user=user, window_size=window_size)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/screenshots
.py", line 74, in get_screenshot
self.screenshot = driver.get_screenshot(self.url, self.element, user)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/webdriver.p
y", line 112, in get_screenshot
driver = self.auth(user)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/webdriver.p
y", line 85, in auth
driver = self.create()
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/webdriver.p
y", line 82, in create
return driver_class(**kwargs)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/firefox
/webdriver.py", line 178, in __init__
RemoteWebDriver.__init__(
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/
webdriver.py", line 269, in __init__
self.start_session(capabilities, browser_profile)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 360, in start_session
response = self.execute(Command.NEW_SESSION, parameters)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 425, in execute
self.error_handler.check_response(response)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 247, in check_response
raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message: Connection refused (os error 111)
[2022-03-03 15:35:56,667: WARNING/ForkPoolWorker-1] Failed at generating thumbnail Message: Connection refused (os error 111)
Traceback (most recent call last):
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/screenshots.py", line 152, in compute_and_cache
payload = self.get_screenshot(user=user, window_size=window_size)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/screenshots.py", line 74, in get_screenshot
self.screenshot = driver.get_screenshot(self.url, self.element, user)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/webdriver.py", line 112, in get_screenshot
driver = self.auth(user)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/webdriver.py", line 85, in auth
driver = self.create()
File "/XXX/XXXsuperset/lib/python3.8/site-packages/superset/utils/webdriver.py", line 82, in create
return driver_class(**kwargs)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/firefox/webdriver.py", line 178, in __init__
RemoteWebDriver.__init__(
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 269, in __init__
self.start_session(capabilities, browser_profile)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 360, in start_session
response = self.execute(Command.NEW_SESSION, parameters)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/webdriver.py", line 425, in execute
self.error_handler.check_response(response)
File "/XXX/XXXsuperset/lib/python3.8/site-packages/selenium/webdriver/remote/errorhandler.py", line 247, in check_response
raise exception_class(message, screen, stacktrace)