centos superset安装

一.安装redis

# redis6.2.6安装
yum install gcc
# 安装依赖
yum install -y gcc
# 新建安装目录,进到相应目录
mkdir /data/Downloads
cd /data/Downloads
# 下载
wget https://download.redis.io/releases/redis-6.2.6.tar.gz
# 下载完毕后开始解压
tar -zxvf redis-6.2.6.tar.gz

# 编译并安装
cd redis-6.2.6
make && make install PREFIX=/usr/local/redis
cd /usr/local/redis
# 新建日志目录
mkdir logs

# 将配置文件复制相应的目录
cp /data/Downloads/redis-6.2.6/redis.conf  /usr/local/redis/bin
cd ./bin

# 修改redis配置
cat redis.conf | grep -v "#" | grep -v "^$" > redis-12389.conf

port 12389
pidfile /var/run/redis_12389.pid
dbfilename dump_12389.rdb
requirepass requirepass
logfile "/usr/local/redis/logs/redis.log"
maxclients 100000
stop-writes-on-bgsave-error no
# 由于改为supervisor监控
# daemonize no


# 启动redis
./redis-server /usr/local/redis/bin/redis-12389.conf
ln -s  /usr/local/redis/bin/redis-cli     /usr/bin/redis-cli

二.安装mysql

略

# 创建superset库
CREATE DATABASE `superset` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;

三.superset安装

3.1安装superset依赖

sudo yum install -y gcc gcc-c++ libffi-devel python3-devel python3-pip python3-wheel openssl-devel cyrus-sasl-devel openldap-devel
pip3 install --upgrade setuptools pip
pip3 install flower
pip3 install flask
pip3 install redis
pip3 install pymysql
pip3 install apache-superset
yum install -y nginx
pip3 install uwsgi
pip3 install gunicorn
# mysql依赖
yum install -y mysql-devel
pip3 install mysqlclient
# hive依赖
pip3 install 'pyhive[hive]'

3.2superset配置路径

mkdir -p /opt/superset/conf
# 将配置文件拷贝到配置路径
cp /usr/local/lib/python3.7/site-packages/superset/config.py /opt/superset/conf/superset_config.py
cd /opt/superset/conf/

# 设置superset路径 修改环境变量
vim /etc/profile
export SUPERSET_CONFIG_PATH=/opt/superset/conf/superset_config.py
source /etc/profile

# 初始化数据库
superset db upgrade

# 通过运行以下命令完成安装
# 创建管理员用户 (在设置密码之前，系统会提示您设置用户名、姓和名)
# 设置FLASK_APP环境变量
export FLASK_APP=superset
superset fab create-admin

# 加载一些数据来展示
# superset load_examples

# 创建默认角色和权限
superset init

# 在端口8088上启动一个开发web服务器, 使用-p绑定到其他端口(调试模式启动)
# superset run -p 8088 --with-threads --reload --debugger

superset_config.py 相关配置

# Superset specific config
from datetime import timedelta
from flask import session
from flask import Flask
from celery.schedules import crontab

ROW_LIMIT = 5000

SUPERSET_WEBSERVER_PORT = 8088

# Flask App Builder configuration
# Your App secret key
SECRET_KEY = '\2\1thisismyscretkey\1\2\e\y\y\h'

# The SQLAlchemy connection string to your database backend
# This connection defines the path to the database that stores your
# superset metadata (slices, connections, tables, dashboards, ...).
# Note that the connection information to connect to the datasources
# you want to explore are managed directly in the web UI
# SQLALCHEMY_DATABASE_URI = 'sqlite:path/to/superset.db'
SQLALCHEMY_DATABASE_URI = 'mysql://user:[email protected]/superset'

# Flask-WTF flag for CSRF
WTF_CSRF_ENABLED = True
# Add endpoints that need to be exempt from CSRF protection
WTF_CSRF_EXEMPT_LIST = []
# A CSRF token that expires in 1 year
WTF_CSRF_TIME_LIMIT = 60 * 60 * 24 * 365

# Set this API key to enable Mapbox visualizations
MAPBOX_API_KEY = ''


def make_session_permanent():
    '''
    Enable maxAge for the cookie 'session'
    '''
    session.permanent = True


# Set up max age of session to 24 hours
PERMANENT_SESSION_LIFETIME = timedelta(hours=24)


def FLASK_APP_MUTATOR(app: Flask) -> None:
    app.before_request_funcs.setdefault(None, []).append(make_session_permanent)

# 标志功能
FEATURE_FLAGS = {
    'CLIENT_CACHE': False,
    'ENABLE_EXPLORE_JSON_CSRF_PROTECTION': False,
    'PRESTO_EXPAND_DATA': False,
    # 缓存缩略图
    "THUMBNAILS": True,
    "THUMBNAILS_SQLA_LISTENERS": True,
}
THUMBNAIL_CACHE_CONFIG = {
    "CACHE_TYPE": "redis",
    "CACHE_NO_NULL_WARNING": False,
    "CACHE_NO_NULL_WARNING": False,
    'CACHE_REDIS_PORT': 12389,  # 配置端口号
    'CACHE_REDIS_URL': 'redis://:[email protected]:12389'  # 配置 URL
}
CACHE_CONFIG = {
    'CACHE_TYPE': 'redis',  # 使用 Redis
    'CACHE_REDIS_HOST': '127.0.0.1',  # 配置域名
    'CACHE_REDIS_PORT': 12389,  # 配置端口号
    'CACHE_REDIS_URL': 'redis://:[email protected]:12389'  # 配置 URL
}
# 缓存
# 对于图表数据Superset 上升了一个“超时搜索路径”，从切片的配置到数据源、数据库的配置，然后最终回退到 中定义的全局默认值DATA_CACHE_CONFIG
DATA_CACHE_CONFIG = {
    'CACHE_TYPE': 'redis',
    'CACHE_DEFAULT_TIMEOUT': 60 * 60 * 24,  # 1 day default (in secs)
    'CACHE_KEY_PREFIX': 'superset_results',
    'CACHE_REDIS_URL': 'redis://:[email protected]:12389/0',
}


# 设置CELERY
class CeleryConfig:  # pylint: disable=too-few-public-methods
    # BROKER_URL = "sqla+sqlite:///celerydb.sqlite"
    # 设置broker的信息
    # 其中password是密码，密码前必须加冒号
    BROKER_URL = 'redis://:[email protected]:12389/0'
    CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks")
    # 设置结果存储信息
    CELERY_RESULT_BACKEND = 'redis://:[email protected]:12389/1'
    CELERYD_LOG_LEVEL = "DEBUG"
    CELERYD_PREFETCH_MULTIPLIER = 1
    CELERY_ACKS_LATE = False
    CELERY_ANNOTATIONS = {
        "sql_lab.get_sql_results": {"rate_limit": "100/s"},
        "email_reports.send": {
            "rate_limit": "1/s",
            "time_limit": 120,
            "soft_time_limit": 150,
            "ignore_result": True,
        },
    }
    # CELERYBEAT_SCHEDULE = {
    #     "email_reports.schedule_hourly": {
    #         "task": "email_reports.schedule_hourly",
    #         "schedule": crontab(minute=1, hour="*"),
    #     }
    # }
    # Superset 有一个 Celery 任务，它会根据不同的策略定期预热缓存。
    # #这将每小时缓存前 5 个最受欢迎的仪表板中的所有图表。对于其他策略，请检查superset/tasks/cache.py文件
    CELERYBEAT_SCHEDULE = {
        'cache-warmup-hourly': {
            'task': 'cache-warmup',
            'schedule': crontab(minute=0, hour='*'),  # hourly
            'kwargs': {
                'strategy_name': 'top_n_dashboards',
                'top_n': 5,
                'since': '7 days ago',
            },
        },
    }


CELERY_CONFIG = CeleryConfig  # pylint: disable=invalid-name

# 设置默认语种
BABEL_DEFAULT_LOCALE = "zh"
# Your application default translation path
BABEL_DEFAULT_FOLDER = "superset/translations"
# The allowed translation for you app
LANGUAGES = {
    # "en": {"flag": "us", "name": "English"},
    #
    # "es": {"flag": "es", "name": "Spanish"},
    # "it": {"flag": "it", "name": "Italian"},
    # "fr": {"flag": "fr", "name": "French"},
    "zh": {"flag": "cn", "name": "Chinese"},
    # "ja": {"flag": "jp", "name": "Japanese"},
    # "de": {"flag": "de", "name": "German"},
    # "pt": {"flag": "pt", "name": "Portuguese"},
    # "pt_BR": {"flag": "br", "name": "Brazilian Portuguese"},
    # "ru": {"flag": "ru", "name": "Russian"},
    # "ko": {"flag": "kr", "name": "Korean"},
}

GLOBAL_ASYNC_QUERIES_REDIS_CONFIG = {
    "port": 12389,
    "host": "127.0.0.1",
    "password": "password",
    "db": 0,
}

3.3superset启动

# 使用gunicorn启动
# gunicorn相关配置参考:https://docs.gunicorn.org/en/stable/design.html
# 如果不使用Gunicorn,通过在superset_config.py中设置COMPRESS_REGISTER = False 来禁用flask-compress
# 16666端口需要开放tcp以及udp
gunicorn -w 10  -k gevent --timeout 120 -b  0.0.0.0:16666 --limit-request-line 0 --limit-request-field_size 0 "superset.app:create_app()"

# 要启动 Celery Worker 以利用配置，请运行以下命令
celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4

# 要启动一个计划定期后台作业的作业，请运行以下命令
celery --app=superset.tasks.celery_app:app beat

四.mysql,hive可视化展示

4.1mysql可视化展示

4.1.1添加mysql连接,添加mysql数据集(Datasets)

五.superset的supervisor配置文件

[program:superset_celery_worker]
environment=FLASK_APP="superset",SUPERSET_CONFIG_PATH="/opt/superset/conf/superset_config.py"
command     = /usr/local/bin/celery --app=superset.tasks.celery_app:app worker --pool=prefork -O fair -c 4
user        = root
startsecs   = 3
redirect_stderr         = true
stdout_logfile_maxbytes = 550MB
stdout_logfile_backups  = 10
stdout_logfile          = /var/lib/docker/containers/facebook_google/superset_celery_worker.log

参考文档 https://superset.apache.org/
参考配置文档 https://github.com/apache/superset/blob/master/superset/config.py
数据库驱动参考https://superset.apache.org/docs/databases/installing-database-drivers