通过gitlab api进行数据迁移脚本

环境

python3
gitlab 8（centos6）
gitlab 11.5（centos7）
数据由gitlab 8（api v3）迁移至gitlab 11.5（api v4）

用户，组，工程数据迁移python脚本

move-data.py

#!/usr/bin/python
# -*- coding:UTF-8 -*-
__author__ = 'you227'
import requests,json

old_gitlab_token = "sdfdkjxingdfdfddfdfdf"
new_gitlab_token = "sdefjxidnginxdfdfddss"
old_git_address = "http://192.168.0.2"#api version v3
new_git_address = "http://192.168.0.3"#api version v4

def getUserData():
    print("开始获取用户信息")
    #调用api获取用户数据
    resultAll=[]
    flag=True
    num=1
    while flag:
        resultJson = httpGet(old_git_address+"/api/v3/users", {"private_token":old_gitlab_token,"page":num,"per_page":100})
        if str(resultJson)=="[]":
            flag=False
        num+=1
        resultAll = resultAll + resultJson
    for line in resultAll:
        if line['name']=='Administrator':
            break
        print('user:'+line['username'])
        findR = getUsersInfo(line['username'])
        print("新gitlab用户:"+str(findR))

        if str(findR)=='[]':
            password = line['username']+"12345"
            adminFlag="false"
            if line['is_admin']:
                adminFlag="true"
            createJson = httpPost(new_git_address+"/api/v4/users", {"private_token":new_gitlab_token,"email":line['email'],"password":password,"username":line['username'],"name":line['name'],"skip_confirmation":"true", "admin":adminFlag})
            print(createJson)

            #修改密码 确保弹出修改密码页面
            ''' updateR = httpPut(new_git_address+"/api/v4/users/"+str(createJson['id']), {"private_token":new_gitlab_token,"password":password})
            print(updateR) '''

            if line['state']=='blocked':
                blockR = httpPost(new_git_address+"/api/v4/users/"+str(createJson['id'])+"/block", {"private_token":new_gitlab_token,"page":num,"per_page":100})
                print(blockR)
        else:
            print(line['username']+',用户已存在')
            #break
    print("获取用户信息成功")

def getGroupData():
    print("开始获取组信息")
    #调用api获取组数据
    resultAll=[]
    flag=True
    num=1
    while flag:
        resultJson = httpGet(old_git_address+"/api/v3/groups", {"private_token":old_gitlab_token,"page":num,"per_page":100})
        if str(resultJson)=="[]":
            flag=False
        num+=1
        resultAll = resultAll + resultJson
    for line in resultAll:
        path = line['path']
        print('开始导出组:'+path)
        createJson = httpPost(new_git_address+"/api/v4/groups", {"private_token":new_gitlab_token,"name":line['name'],"path":path,"description":line['description'],"visibility":"public"})
        print("导出组返回信息："+str(createJson))
        #获取组下用户信息准备添加到组
        findJson = getGroupUsersData(path, line['id'])
        for user in findJson:
            userInfo = getUsersInfo(user['username'])
            if str(userInfo)=="[]":
                print(user['username']+"用户不存在")
            else:
                addUsersToGroup(path, userInfo[0]['id'], user['username'], user['access_level'])
    print("获取组信息成功")

def getProjectData():
    print("开始获取工程信息")
    #调用api获取工程数据
    resultAll=[]
    flag=True
    num=1
    while flag:
        resultJson = httpGet(old_git_address+"/api/v3/projects/all", {"private_token":old_gitlab_token,"page":num,"per_page":1})
        if str(resultJson)=="{'message': '500 Internal Server Error'}":
            print("error num:%d" % (num))
        else:
            if str(resultJson)=="[]":
                flag=False
            resultAll = resultAll + resultJson
        num+=1
    #print(len(resultAll))
    #提取数据保存到csv
    csv_file = open("project.csv", "w", encoding='utf-8')
    csv_data = ''
    data_num=0
    for line in resultAll:
        if data_num!=0:
            csv_data+='\n'
        csv_data+=line['namespace']['path']+','+line['path']+','+line['name']
        data_num+=1
    print("工程总数:"+str(data_num))
    csv_file.write(csv_data)
    csv_file.close()
    #调用api发送数据到新gitlab
    for line in resultAll:
        print("工程开始：namespace_path:%s, name:%s,path:%s" % (line['namespace']['path'], line['name'], line['path']))
        namespace = getNamspaceData(line['namespace']['path'])
        visibility_level = line['visibility_level']
        visibility = "private"
        if visibility_level == 10:
            visibility = "internal"
        elif visibility_level == 20:
            visibility = "public"
        if namespace is None:
            continue
        projectInfo = getProjectInfo(line['path'], line['namespace']['path'])
        print("projectInfo:"+str(projectInfo))
        if projectInfo is None:
            projectInfo = httpPost(new_git_address+"/api/v4/projects", {"private_token":new_gitlab_token,"name":line['name'],"path":line['path'],"namespace_id":namespace['id'],"description":line['description'],"visibility":visibility})
        print("createInfo:"+str(projectInfo))
        findJson = getProjectUsersData(line['path_with_namespace'], line['id'])
        for user in findJson:
            userInfo = getUsersInfo(user['username'])
            if str(userInfo)=="[]":
                print(user['username']+"用户不存在")
            else:
                print("给用户添加工程权限"+line['path_with_namespace'])
                addUsersToProject(projectInfo['id'], userInfo[0]['id'], user['username'], user['access_level'])
    print("获取工程信息成功")

def getGroupUsersData(group_path, group_id):
    print("开始获取"+group_path+"组下所有用户信息")
    #调用api获取数据
    resultJson = httpGet(old_git_address+"/api/v3/groups/"+str(group_id)+"/members", {"private_token":old_gitlab_token,"per_page":100})
    print(resultJson)
    print("获取"+group_path+"组下所有用户信息成功")
    return resultJson

def getUsersInfo(username):
    print("开始获取"+username+"用户信息")
    #调用api获取数据
    resultJson = httpGet(new_git_address+"/api/v4/users", {"private_token":new_gitlab_token,"username":username})
    print(resultJson)
    print("获取"+username+"用户信息成功")
    return resultJson

def addUsersToGroup(group_path, user_id, username, access_level):
    print("将用户加入%s组,%d:%s用户,%s级别" % (group_path, user_id, username, access_level))
    #调用api获取数据
    resultJson = httpPost(new_git_address+"/api/v4/groups/"+group_path+"/members", {"private_token":new_gitlab_token,"user_id":user_id,"access_level":access_level})
    print(resultJson)
    print("将用户加入组成功")
    return resultJson

def getProjectUsersData(project_path, project_id):
    print("开始获取"+project_path+"工程下所有用户信息")
    #调用api获取数据
    resultJson = httpGet(old_git_address+"/api/v3/projects/"+str(project_id)+"/members", {"private_token":old_gitlab_token,"per_page":100})
    print(resultJson)
    print("获取"+project_path+"工程下所有用户信息成功")
    return resultJson

def addUsersToProject(project_id, user_id, username, access_level):
    print("将用户加入%d工程,%d:%s用户,%s级别" % (project_id, user_id, username, access_level))
    #调用api获取数据
    resultJson = httpPost(new_git_address+"/api/v4/projects/"+str(project_id)+"/members", {"private_token":new_gitlab_token,"user_id":user_id,"access_level":access_level})
    print(resultJson)
    print("将用户加入工程成功")
    return resultJson

def getNamspaceData(path):
    print("开始获取"+path+"工作空间信息")
    #调用api获取数据
    resultJson = httpGet(new_git_address+"/api/v4/namespaces", {"private_token":new_gitlab_token,"per_page":100, "search":path})
    print(resultJson)
    if len(resultJson) > 0:
        for line in resultJson:
            if path==line['path']:
                print("获取"+path+"工作空间成功")
                return line
        print("获取"+path+"工作空间失败，未找到工作空间")
    else:
        print("获取"+path+"工作空间失败，返回值为空")

def getProjectInfo(path, namespace_path):
    print("开始获取"+path+"工程信息")
    #调用api获取数据
    resultJson = httpGet(new_git_address+"/api/v4/projects", {"private_token":new_gitlab_token,"per_page":100, "search":path})
    print(resultJson)
    if len(resultJson) > 0:
        for line in resultJson:
            if namespace_path==line['namespace']['path'] and path==line['path']:
                print("获取"+path+"工程成功")
                return line
        print("获取"+path+"工程失败，未找到工程")
    else:
        print("获取"+path+"工程失败，返回值为空")

def httpGet(url, data):
    result = requests.get(url, data=data)
    result.encoding = 'utf-8'
    resultJson = json.loads(result.text)
    return resultJson

def httpPost(url, data):
    result = requests.post(url, data=data)
    result.encoding = 'utf-8'
    resultJson = json.loads(result.text)
    return resultJson

def httpPut(url, data):
    result = requests.put(url, data=data)
    result.encoding = 'utf-8'
    resultJson = json.loads(result.text)
    return resultJson

#批量修改组权限
def updateGroupData():
    print("开始获取组信息")
    #调用api获取组数据
    resultAll=[]
    flag=True
    num=1
    while flag:
        resultJson = httpGet(new_git_address+"/api/v4/groups", {"private_token":new_gitlab_token,"page":num,"per_page":100})
        if str(resultJson)=="[]":
            flag=False
        num+=1
        resultAll = resultAll + resultJson
    print(len(resultAll))
    for line in resultAll:
        createJson = httpPut(new_git_address+"/api/v4/groups/"+str(line['id']), {"private_token":new_gitlab_token,"visibility":"public"})
        print(createJson)


getUserData()
getGroupData()
getProjectData()

用户密码还原

1.登录postgresql数据库导出用户密码数据
2.通过脚本生成修改密码sql
3.执行密码修改sql

#查看用户
cat /etc/passwd
#进入gitlab数据库
docker exec -it gitlab bash
su - gitlab-psql
psql -h /var/opt/gitlab/postgresql -d gitlabhq_production
#查看帮助命令
\h
#查看数据库
\l
#查看多表
\dt
#查看单表
\d abuse_reports
#退出psql
\q
#导出用户密码信息
\COPY (select username,encrypted_password from users) to '/home/postgres/users.csv' with csv header;
#导入sql文件
cp update-password.sql /gitlab/data
docker exec -it gitlab bash
su - gitlab-psql
psql -h /var/opt/gitlab/postgresql -d gitlabhq_production < /var/opt/gitlab/update-password.sql

生成密码修改sql python脚本

updateSqlMake.py

#!/usr/bin/python
# -*- coding:UTF-8 -*-
__author__ = 'youxiang'

sql=''
for line in open("users.csv", encoding='utf-8'):
    line=line.strip('\n')
    arr = line.split(',')
    update_sql="update users set encrypted_password='%s' where username='%s';\n" % (arr[1], arr[0])
    sql = sql + update_sql
sql_file=open("update-password.sql", 'w', encoding='utf-8')
sql_file.write(sql)
sql_file.close()

代码迁移shell脚本

move-code.sh

new_gitlab_url="http://192.168.0.3"
old_gitlab_url="http://192.168.0.2"
file="project.csv"
localpath=$(cd `dirname $0`; pwd)
cd ${localpath}
for line in `cat $file`
do
    array=(${line//,/ })
    namespace=${array[0]}
    path=${array[1]}
    new=${new_gitlab_url}"/"${namespace}"/"${path}".git"
    old=${old_gitlab_url}"/"${namespace}"/"${path}".git"
    echo $new
    echo $old
    git clone $old
    cd ${path}
    #获取所有远程分支
    git branch -r | grep -v '\->' | while read remote; do git branch --track "${remote#origin/}" "$remote"; done
    #更新所有本地分支
    git branch | awk 'BEGIN{print "echo ****Update all local branch...@daimon***"}{if($1=="*"){current=substr($0,3)};print a"git checkout "substr($0,3);print "git pull --all";}END{print "git checkout " current}' | sh
    #修改源库，并推送到新仓库
    git remote rename origin old-origin
    #新工程git地址
    git remote add origin $new
    git push -u origin --all
    git push -u origin --tags
    cd ..
    #删除下载的目录
    #rm -rf ${path}
    echo "${array[0]},${array[1]},${array[2]}" >> over_project.csv
done

数据校验python脚本

checkProjectAndBranch.py

#!/usr/bin/python
# -*- coding:UTF-8 -*-
__author__ = 'you227'
import requests,json

old_gitlab_token = "sdfdkjxingdfdfddfdfdf"
new_gitlab_token = "sdefjxidnginxdfdfddss"
old_git_address = "http://192.168.0.2"#api version v3
new_git_address = "http://192.168.0.3"#api version v4

def checkProjectBranch():
    print("开始获取工程信息")
    #调用api获取工程数据
    resultAll=[]
    flag=True
    num=1
    while flag:
        resultJson = httpGet(old_git_address+"/api/v3/projects/all", {"private_token":old_gitlab_token,"page":num,"per_page":1})
        if str(resultJson)=="{'message': '500 Internal Server Error'}":
            print("error num:%d" % (num))
        else:
            if str(resultJson)=="[]":
                flag=False
            resultAll = resultAll + resultJson
        num+=1
    #print(len(resultAll))
    #提取数据
    project_data=[]
    for line in resultAll:
        project_data.append(line['namespace']['path']+','+line['path']+','+line['name'])
    #获取新gitlab工程数据
    new_project_data = getNewProjectData()
    #比对工程数据
    for line in project_data:
        if line not in new_project_data:
            print("新gitlab找不到工程:"+line)
    #调用api发送数据到新gitlab
    for line in resultAll:
        print("工程开始：namespace_path:%s, name:%s,path:%s" % (line['namespace']['path'], line['name'], line['path']))
        projectInfo = getProjectInfo(line['path'], line['namespace']['path'])
        print("newProjectInfo:"+str(projectInfo))
        if projectInfo is None:
            print("检查分支时新gitlab没找到工程:"+line['path_with_namespace'])
            continue
        findJson = getOldProjectBranches(line['id'])
        #print("oldProjectBranchInfo:"+str(findJson))
        findNewJson = getNewProjectBranches(projectInfo['id'])
        #print("newProjectBranchInfo:"+str(findNewJson))
        for branch in findJson:
            oldBranchName = branch['name']
            oldCommitId=branch['commit']['id']
            branchFlag=True
            commitFlag=True
            for newBranch in findNewJson:
                if oldBranchName == newBranch['name']:
                    branchFlag = False
                if oldCommitId == newBranch['commit']['id']:
                    commitFlag = False
            if branchFlag:
                print("工程:"+line['path_with_namespace']+"没找到分支:"+oldBranchName)
            if commitFlag:
                print("工程:"+line['path_with_namespace']+"没找到commit:"+oldCommitId)
    print("对比工程信息完成")

def getProjectUsersData(project_path, project_id):
    print("开始获取"+project_path+"工程下所有用户信息")
    #调用api获取数据
    resultJson = httpGet(old_git_address+"/api/v3/projects/"+str(project_id)+"/members", {"private_token":old_gitlab_token,"per_page":100})
    print(resultJson)
    print("获取"+project_path+"工程下所有用户信息成功")
    return resultJson

def getProjectInfo(path, namespace_path):
    print("开始获取"+path+"工程信息")
    #调用api获取数据
    resultJson = httpGet(new_git_address+"/api/v4/projects", {"private_token":new_gitlab_token,"per_page":100, "search":path})
    print(resultJson)
    if len(resultJson) > 0:
        for line in resultJson:
            if namespace_path==line['namespace']['path'] and path==line['path']:
                print("获取"+path+"工程成功")
                return line
        print("获取"+path+"工程失败，未找到工程")
    else:
        print("获取"+path+"工程失败，返回值为空")

def getNewProjectBranches(projectId):
    print("开始获取新工程分支信息")
    #调用api获取工程数据
    resultJson = httpGet(new_git_address+"/api/v4/projects/"+str(projectId)+"/repository/branches", {"private_token":new_gitlab_token,"per_page":100})
    print(len(resultJson))
    print("获取新工程分支信息完成")
    return resultJson

def getOldProjectBranches(projectId):
    print("开始获取工程分支信息")
    #调用api获取工程数据
    resultJson = httpGet(old_git_address+"/api/v3/projects/"+str(projectId)+"/repository/branches", {"private_token":old_gitlab_token,"per_page":100})
    print(len(resultJson))
    print("获取工程分支信息完成")
    return resultJson

def getNewProjectData():
    print("开始获取工程信息")
    #调用api获取工程数据
    resultAll=[]
    flag=True
    num=1
    while flag:
        resultJson = httpGet(new_git_address+"/api/v4/projects", {"private_token":new_gitlab_token,"page":num,"per_page":100})
        if str(resultJson)=="[]":
            flag=False
        resultAll = resultAll + resultJson
        num+=1
    print(len(resultAll))
    #提取数据保存到csv
    project_data = []
    for line in resultAll:
        project_data.append(line['namespace']['path']+','+line['path']+','+line['name'])
    print("获取工程信息成功")
    return project_data

def httpGet(url, data):
    result = requests.get(url, data=data)
    result.encoding = 'utf-8'
    resultJson = json.loads(result.text)
    return resultJson

def httpPost(url, data):
    result = requests.post(url, data=data)
    result.encoding = 'utf-8'
    resultJson = json.loads(result.text)
    return resultJson

def httpPut(url, data):
    result = requests.put(url, data=data)
    result.encoding = 'utf-8'
    resultJson = json.loads(result.text)
    return resultJson

checkProjectBranch()

gitlab常用命令

#管理gitlab
docker exec -t gitlab bash
# Start GitLab
gitlab-ctl start
# Stop GitLab
gitlab-ctl stop
# Restart GitLab
gitlab-ctl restart
vi /etc/gitlab/gitlab.rb
gitlab_rails['gitlab_shell_ssh_port'] = 222
#备份定时清理，最长保留7天
gitlab_rails['backup_keep_time'] = 604800
#重新加载gitlab的配置文件
gitlab-ctl reconfigure
#备份
docker exec -t gitlab gitlab-rake gitlab:backup:create
#恢复
gitlab-rake gitlab:backup:restore BACKUP=1508412719_2017_10_19_10.0.2

定时备份

crontab -e
#输入下面内容
0 12 * * * `sh /gitlab/gitlab-backup/gitlab-backup.sh`
0 20 * * * `sh /gitlab/gitlab-backup/gitlab-backup.sh`
#同步服务器时间
*/10 * * * * /usr/sbin/ntpdate 192.168.0.100;/sbin/clock -w

#参数
minute   hour   day   month   week   command
其中：
minute： 表示分钟，可以是从0到59之间的任何整数。
hour：表示小时，可以是从0到23之间的任何整数。
day：表示日期，可以是从1到31之间的任何整数。
month：表示月份，可以是从1到12之间的任何整数。
week：表示星期几，可以是从0到7之间的任何整数，这里的0或7代表星期日。
command：要执行的命令，可以是系统命令，也可以是自己编写的脚本文件。
#重启crontab 
systemctl start crontab

gitlab-backup.sh

#!/usr/bin/bash
basepath=$(cd `dirname $0`; pwd)
start_date=`date +"%Y-%m-%d %H:%M:%S"`
echo "******** gitlab开始备份 开始时间:$start_date ********" >> $basepath/backup.log

#开始备份  docker 安装方式
/usr/bin/docker exec -t gitlab gitlab-rake gitlab:backup:create >> $basepath/backup.log

end_date=`date +"%Y-%m-%d %H:%M:%S"`
echo "******** gitlab备份结束 结束时间:$end_date ********" >> $basepath/backup.log
start_time=`date -d "$start_date" '+%M'`
end_time=`date -d  "$end_date" '+%M'`
interval=`expr $end_time - $start_time`
echo "******** gitlab备份时长:"$interval m" ********" >> $basepath/backup.log