CentOS 7下优雅的数据科学环境

0x0 安装阶段配置

1. 网络配置

  • 网卡地址ipv4
  • 子网掩码
  • 网关地址
  • DNS服务器
  • hostname

2. 用户配置

  • 配置密码

0x1 基础环境

1. 网络配置

修改ifcfg配置文件/etc/sysconfig/network-scripts/ifcfg-*

2. 用户配置

  • 创建用户
[root@data ~]# adduser messiah 会有自动生成目录
[root@data ~]# useradd datag 不会生成目录
  • 创建用户组
[root@data ~]# groupadd docker
  • 修改密码
[root@data ~]# passwd messiah
  • 修改sudoers权限
echo 'messiah   ALL=(ALL)   NOPASSWD: ALL' >> /etc/sudoers
cat /etc/sudoers  | grep -Evn "^$|#"
  • 常用命令
  • 删除用户
[root@data ~]# deluser datag 不会删除目录
[root@data ~]# userdel datag 不会删除目录
  • 删除用户组
[root@data ~]# groupdel docker 
  • 看看linux内核
[root@data ~]# uname -a
[root@data ~]# cat /etc/issue

修改开机欢迎
加日期

➜  ~ cat /etc/issue
\S
Kernel \r on an \m
\t \d

加banner

➜  ~ sudo cat /etc/ssh/sshd_config | grep banner
# no default banner path
Banner /etc/ssh/banner
➜  ~ cat /etc/ssh/banner
Welcome to Lefit Data

加字符图

➜  ~ cat /etc/motd
//
//                       .::::.
//                     .::::::::.
//                    :::::::::::
//                 ..:::::::::::'
//              '::::::::::::'
//                .::::::::::
//           '::::::::::::::..
//                ..::::::::::::.
//              ``::::::::::::::::
//               ::::``:::::::::'        .:::.
//              ::::'   ':::::'       .::::::::.
//            .::::'      ::::     .:::::::'::::.
//           .:::'       :::::  .:::::::::' ':::::.
//          .::'        :::::.:::::::::'      ':::::.
//         .::'         ::::::::::::::'         ``::::.
//     ...:::           ::::::::::::'              ``::.
//    ```` ':.          ':::::::::'                  ::::..
//                       '.:::::'                    ':'````..
欢迎光临Lefit Data

开机自启动

➜  ~ sudo chmod u+x /etc/rc.d/rc.local
➜  ~ sudo chmod u+x /etc/rc.local
➜  ~ sudo chmod +x /home/messiah/documents/scripts/sh/start_int.sh
➜  ~ cat documents/scripts/sh/start_int.sh | grep -Ev "^$|#"
➜  sudo rsync /var/spool/cron/messiah  /home/messiah/documents/git/databack/sys/messiah
su messiah
cd ~
source .zshrc
docker start me_mysql
docker start me_zabbixs
nohup jupyter notebook --config .jupyter/me.py --allow-root 2>&1 >> logs/me.log &
nohup jupyter notebook --config .jupyter/datag.py --allow-root 2>&1 >> logs/datag.log &
➜  ~ cat /etc/rc.d/rc.local | grep -Ev "^$|#"
touch /var/lock/subsys/local
su messiah -c "/home/messiah/documents/scripts/sh/start_int.sh"
➜  ~ cat /etc/ssh/banner
Welcome to Lefit Data

[root@data ~]# cat /proc/version
[root@data ~]# lsb_release -a

- 杀session
```shell
[root@data ~]# pkill -kill -t pts/1
  • du : 显示每个文件和目录的磁盘使用空间~~~文件的大小。
[root@data ~]# du -sh *
  • df:显示磁盘分区上可以使用的磁盘空间
[root@data ~]# df -h
  • free 可以显示Linux系统中空闲的、已用的物理内存及swap内存,及被内核使用的buffer。
[root@data ~]# free
  • fdisk磁盘操作
[root@data ~]# fdisk -l
  • 查看端口占用
[root@data ~]# netstat -tunlp
  • ps命令——查看静态的进程统计信息(Processes Statistic)
[root@data ~]# ps aux
[root@data ~]# ps elf
  • 查看启动项
[root@data ~]# systemctl list-unit-files
  • 远程scp复制文件
[root@data ~]# scp me.zip  [email protected]:~

3. 初始化脚本

#!/bin/bash
#
echo 配置就绪

function pubkey(){
        echo 配置ssh免密登陆不用root;
        mkdir -vpm 700 $HOME/.ssh ;
        echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+7OCqPFwJ560E8JdfQEV6MDqBM6yITXzHJbRQMPG/yy59t+Eb3A8lsxnJ8BMlfmvkJJMtPYJrnuoCeBnVwQlpTA3OT+MK7p0GR2BxKKdaygqumucaAdaFZgBEyQhD4zZJ+zAv6myb4ePlqkms6dq1wnf+Jn5uvD3X1JV9+i/C+L9nRfGIZXSY5viCofktXN9imOtPeQ01XbbFdjx0ShvHgGeMXtqumtLRs8wgGnymH+brDSi+62Cid6KkwjhH2DL6qAFpUD47kW8yRqGAG/bH6ny47lwWuVcuMYtOcOatfezhWcS5Uhw5PtfODXrYEC/Gj4T1rG5f5ocKjvyxIzQ/ [email protected]"\
        >> $HOME/.ssh/authorized_keys  && chmod 600 $HOME/.ssh/authorized_keys;
}

function iniinstall(){
    echo 安装基础依赖需要root;
    yum -y install git readline* openssl* sqlite* sudo bzip2* gcc-c++ gcc zlib-devel openssh* ntp ntpdate net-tools psmisc bash-completion lsb_release;
    yum -y install redhat-lsb policycoreutils-python nmap glibc-common pciutils usbutils hdparm crontabs wget vim curl lynx tree lsof unixODBC zip unzip sendmail sendmail-cf m4;
    yum -y install glibc e2fsprogs krb5-libs unixODBC unixODBC-devel python-devel python-pip python-wheel  libffi-devel libsasl2-devel openldap-devel;
    yum remove -y docker docker-common docker-selinux docker-engine;
    rm -rf /var/lib/docker;
    yum install -y yum-utils device-mapper-persistent-data lvm2;
    yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo;
    yum install -y docker go;
    yum update -y && yum -y install zsh;
}

function test(){
        echo test function;
}


if [ "$1" == "rootinit" ]
then
    iniinstall
  
elif [ "$1" == "pubkey" ]
then
    pubkey

elif [ "$1" == "test" ]
then
        test
fi
  • 配置ssh免密登陆
[messiah@data ~]$ sh init.sh  pubkey
  • 安装基础依赖
[messiah@data ~]$ sudo sh init.sh rootinit

4. 常用软件

  • 安装zsh
[messiah@data ~]$ sh -c "$(curl -fsSL https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"
➜  ~ echo 'export PATH="'$HOME'/.pyenv/bin:$PATH"
eval "$(pyenv init -)"
eval "$(pyenv virtualenv-init -)"' >> $HOME/.zshrc
➜  ~ curl -L https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer | bash;

0x2 开发环境

1. py环境配置

➜  ~ pyenv install 3.8.2
➜  ~ pyenv rehash
➜  ~ pyenv global 3.8.2
  • 常用命令
➜  ~ pyenv install --list # 列出可安装版本
➜  ~ pyenv install  # 安装对应版本
➜  ~ pyenv install -v  # 安装对应版本,若发生错误,可以显示详细的错误信息
➜  ~ pyenv versions # 显示当前使用的python版本
➜  ~ pyenv which python # 显示当前python安装路径
➜  ~ pyenv global  # 设置默认Python版本
➜  ~ pyenv local  # 当前路径创建一个.python-version, 以后进入这个目录自动切换为该版本
➜  ~ pyenv shell  # 当前shell的session中启用某版本,优先级高于global 及 local
➜  ~ pyenv uninstall 3.8.2  # 卸载特定版本
➜  ~ pyenv rehash  # 刷新shims
➜  ~ pyenv virtualenv 2.7.13 myenv  # 创建虚拟环境
➜  ~ pyenv virtualenvs  # 创建虚拟环境
➜  ~ pyenv uninstall myenv  # 删除虚拟环境
➜  ~ pyenv activate myenv  # 切换到虚拟环境
➜  ~ source deactivate  # 退出虚拟环境
  • 从requirements.txt安装依赖库
➜  ~ pip install --upgrade pip
➜  ~ pip install -r requirements.txt
  • 生成pip依赖库文件
➜  ~ pip freeze > requirements.txt
  • 配置jupyter
# jupyter静态文件目录
➜  ~ ./.pyenv/versions/3.6.7/lib/python3.6/site-packages/notebook/static/base/images/
➜  ~ jupyter notebook --generate-config
➜  ~ cat datag.py
c.NotebookApp.allow_remote_access = True
c.NotebookApp.ip = '*'
c.NotebookApp.open_browser = False
c.NotebookApp.notebook_dir = u'/home/messiah'
c.NotebookApp.password = u'sha1:2847ace67da3:bd0565fac7836f2419855b40e84f6e38ecd27085'
c.NotebookApp.port = 18888
➜  ~ nohup jupyter notebook --config .jupyter/datag.py --allow-root 2>&1 > logs/datag.log &
➜  ~ sudo firewall-cmd --zone=public --add-port=18888/tcp --permanent
➜  ~ sudo firewall-cmd --reload
  • python 3.6.x 常用依赖库
aliyun-log-cli==0.1.15.2
aliyun-log-python-sdk==0.6.42
aliyun-python-sdk-core==2.13.1
aliyun-python-sdk-core-v3==2.13.3
aliyun-python-sdk-ecs==4.16.1
aliyun-python-sdk-kms==2.6.0
appnope==0.1.0
asn1crypto==0.22.0
attrs==17.2.0
Automat==0.6.0
backcall==0.1.0
bleach==3.1.0
certifi==2018.8.24
cffi==1.10.0
chardet==3.0.4
Click==7.0
constantly==15.1.0
crcmod==1.7
cryptography==1.9
cssselect==1.0.1
cycler==0.10.0
cypari==2.3.1
Cython==0.29.3
dateparser==0.7.0
decorator==4.3.0
defusedxml==0.5.0
docopt==0.6.2
elasticsearch==6.3.1
entrypoints==0.3
fake-useragent==0.1.11
Flask==1.1.1
future==0.18.2
FXrays==1.3.3
graphviz==0.10.1
greenlet==0.4.15
hdfs3==0.3.0
hyperlink==17.2.1
idna==2.5
imageio==2.5.0
incremental==17.5.0
ipykernel==5.1.0
ipython==6.2.1
ipython-genutils==0.2.0
ipywidgets==7.4.2
itsdangerous==1.1.0
jedi==0.13.2
Jinja2==2.10.1
jmespath==0.9.3
jsoncsv==2.0.9
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.2.4
jupyter-console==6.0.0
jupyter-core==4.4.0
kiwisolver==1.0.1
lxml==3.8.0
MarkupSafe==1.1.0
matplotlib==3.0.2
mistune==0.8.4
MyQR==2.3.1
nbconvert==5.4.0
nbformat==4.4.0
networkx==2.4
notebook==5.7.4
numpy==1.14.2
oss2==2.6.1
pandas==0.23.4
pandocfilters==1.4.2
parsel==1.2.0
parso==0.3.1
pexpect==4.6.0
pickleshare==0.7.5
Pillow==6.1.0
plink==2.3
prometheus-client==0.5.0
prompt-toolkit==1.0.15
protobuf==3.6.1
ptyprocess==0.6.0
pyasn1==0.2.3
pyasn1-modules==0.0.9
pycparser==2.17
pycryptodome==3.8.1
PyDispatcher==2.0.5
Pygments==2.3.1
pymongo==3.7.2
pymssql==2.1.4
PyMySQL==0.7.11
pyodbc==4.0.24
pyodps==0.7.19
pyOpenSSL==17.0.0
pyparsing==2.3.1
pypng==0.0.20
python-dateutil==2.7.5
pytz==2018.9
pyzmq==17.1.2
qrcode==6.1
qtconsole==4.4.3
queuelib==1.4.2
regex==2019.1.24
requests==2.19.1
Scrapy==1.4.0
Send2Trash==1.5.0
service-identity==17.0.0
simplegeneric==0.8.1
six==1.10.0
snappy==2.7
snappy-manifolds==1.1
spherogram==1.8.2
SQLAlchemy==1.2.11
terminado==0.8.1
testpath==0.4.2
tornado==5.1.1
tqdm==4.43.0
traitlets==4.3.2
Twisted==17.5.0
tzlocal==1.5.1
urllib3==1.23
w3lib==1.17.0
wcwidth==0.1.7
webencodings==0.5.1
Werkzeug==0.16.0
widgetsnbextension==3.4.2
XlsxWriter==1.1.0
xlwt==1.3.0
zope.interface==4.4.2

2. docker服务

初始化安装已经在init脚本中完成

  • 添加docker用户组
➜  ~ sudo groupadd docker
➜  ~ sudo gpasswd -a ${USER} docker # 将docker添加到用户组管理取消sudo操作
➜  ~ sudo usermod -aG docker $USER # 将docker追加到用户组
➜  ~ newgrp - docker  # 切换用户组
➜  ~ export DOCHOME="/home/messiah/documents/docker" 设置doc快捷变量
  • 禁用内核selinux
➜  ~ sudo vim /etc/sysconfig/docker
➜  ~ cat /etc/sysconfig/docker | grep -Ev "^$|#"
OPTIONS='--selinux-enabled=false --log-driver=journald --signature-verification=false'
if [ -z "${DOCKER_CERT_PATH}" ]; then
    DOCKER_CERT_PATH=/etc/docker
fi
  • 修改镜像源(换源后需要重启)
➜  ~ sudo vim /etc/docker/daemon.json
➜  ~ cat /etc/docker/daemon.json
{"registry-mirrors": [
"https://dockerhub.azk8s.cn",
"https://registry.docker-cn.com",
"http://hub-mirror.c.163.com",
"https://docker.mirrors.ustc.edu.cn",
"https://reg-mirror.qiniu.com"
]}
➜  ~ sudo systemctl daemon-reload
➜  ~ sudo systemctl restart docker
  • 启动docker服务
➜  ~ sudo systemctl start docker
➜  ~ sudo systemctl enable docker
➜  ~ sudo systemctl status -l docker.service
  • 常用命令
➜  ~ docker version # 列出可安装版本
➜  ~ docker info # 安装对应版本
➜  ~ docker search mysql # 从查找docker镜像
➜  ~ docker pull mysql:5.7.29 从源拉取镜像
➜  ~ docker images -a
➜  ~ docker run -p 13306:3306 --name me_mysql -v $DOCHOME/mysql_latest/conf:/etc/mysql/conf.d -v $DOCHOME/mysql_latest/logs:/logs -v $DOCHOME/mysql_latest/data:/var/lib/mysql -e MYSQL_ROOT_PASSWORD=123456 -d mysql:latest
➜  ~ docker ps -a
➜  ~ docker container ls -a
➜  ~ docker container kill [containID]
➜  ~ docker exec -it me_mysql /bin/bash

3. zabbix服务

➜  ~ docker pull j90w/zabbix-server:zabbix-server
➜  ~ mkdir -p $DOCHOME/me_zabbixs/mysql $DOCHOME/me_zabbixs/vhosts/www.me_zabbixs.com
➜  ~ docker run -d -p 10053:10050 -p 10054:10051 -p 18022:22 -p 13316:3306 \
-e PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
-e  container=container -e VIRTUAL_HOST=www.me_zabbixs.com -e USERNAME=messiah \
-e SFTP_PORT=18000 -e MYSQL_PORT=13316 -e [email protected] \
--name me_zabbixs --publish-all -v $DOCHOME/me_zabbixs -v $DOCHOME/me_zabbixs/mysql:/var/lib/mysql \
-v $DOCHOME/me_zabbixs/vhosts/www.me_zabbixs.com:/var/www/html --hostname=www.me_zabbixs.com \
--privileged j90w/zabbix-server:zabbix-server '/usr/sbin/init'
➜  ~ docker exec -it  -u 0 me_zabbixs /bin/bash
访问:http://172.16.150.163:32768/ 基础配置

3. hexo日志服务

➜   ~ docker pull spurin/hexo
➜   ~ docker run -d -p 14000:4000 -p 14022:22 -e HEXO_SERVER_PORT=4000 \
-e ROOT=TRUE -e USER=messiah -e PASSWORD=123456 \
--name me_hexo -v /home/messiah/documents/docker/me_hexo:/home/$USER/hexo \
-e USERID=$UID  spurin/hexo
➜   ~ sudo docker exec -it me_hexo bash
root@2171eb438e35:~# npm install hexo-symbols-count-time
root@2171eb438e35:~# cp -r _config.yml source themes  ../home/messiah/hexo/
root@2171eb438e35:~# ln -s /home/messiah/hexo/_config.yml _config.yml
root@2171eb438e35:~# ln -s /home/messiah/hexo/source source
root@2171eb438e35:~# ln -s /home/messiah/hexo/themes themes
➜   me_hexo git clone https://github.com/theme-next/hexo-theme-next themes/next
➜   me_hexo git clone https://github.com/iissnan/hexo-theme-next themes/iissnan_next
root@2171eb438e35:~# hexo clean && hexo g && hexo 
➜   me_hexo vim themes/next/layout/_partials/footer.swig
➜   md git:(master) ✗ rsync -a /home/messiah/documents/docker/me_hexo/source/ /home/messiah/documents/git/databack/md
➜   md git:(master) ✗ git status
➜   md git:(master) ✗ git add -A :/
➜   md git:(master) ✗ git commit -am "添加hexo目录等"
➜   md git:(master) git push -u origin master

你可能感兴趣的:(CentOS 7下优雅的数据科学环境)