使用Python导入Excel到Sqlite3

       公司一个项目使用SQLite做数据存储，但是SQLite的数据每次修改都比较麻烦，不利于策划配置数值，于是我想到帮策划写一个Python工具，配合上shell，可以在win上进行一件转换，方便之极。

       Python扫盲：RUNOOB   /    廖雪峰的官网

       解析Excel并且生成 .Sql 文件的万能Python脚本，（中文注释，以后有时间再加）

xls2db.py

# coding=utf-8
import sys
reload(sys)
sys.setdefaultencoding( "utf-8" )

from optparse import OptionParser
import os, codecs, xlrd, collections
from jinja2 import Environment, PackageLoader

def parseInt(value):
	value = str(value)
	if value == "":
		value = "0"
	return "%s" % int(float(value))

def parseFloat(value):
	value = str(value)
	if value == "":
		value = "0"
	return "%s" % float(value)

def isUnicode(s):
	return isinstance(s,unicode)

def dataType2SqlType(dataType):
	dataType = dataType.strip()
	ret = None
	if dataType == "string":
		ret = "varchar(32)"
	elif dataType == "int":
		ret = "int"
	elif dataType == "float":
		ret = "float"
	elif dataType == "text":
		ret = "text"
	else:
		print u"要求转换的数据类型是[%s]，错误" % (dataType)
		pass

	return ret

def data2ValueWithType(dataType,data):
	dataType = dataType.strip()
	ret = None
	if dataType == "string":
		ret = "'%s'" % data
	elif dataType == "int":
		ret = parseInt(data)
	elif dataType == "float":
		ret = parseFloat(data)
	elif dataType == "text":
		ret = data
	else:
		print u"数据转化错误 [%s]" % (dataType)
		
	return ret

def convertSheet(sheet,baseDir):
	global tables
	name = sheet.name
	print u"开始导出表:",name
	try:
		if sheet.nrows == 0:
			raise ValueError(u"表没有数据行")

		objname_row = sheet.row(0)
		print u"表名: ",objname_row[0]
		if objname_row[0].value=="":
			raise ValueError(u"表头是空的")

		if objname_row[1].value!="":
			raise ValueError(u"表头不仅仅是一列,是多咧")

		title_row 	= sheet.row(1)
		colname_row = sheet.row(2)
		coltype_row = sheet.row(3)

		ingor_row	= sheet.row(5)

		words = collections.OrderedDict()
		colnames = []
		coltypes = []
		coltitles = []
		#print "Length:",len(colname_row)
		for i in range(0,len(colname_row)):
			colname = colname_row[i].value.strip()
			coltype = coltype_row[i].value.strip()
			coltitle = title_row[i].value.strip()
			if coltype == "" or ingor_row[i].value == 0:
				print u"\033[0;31m [%s]列 列名[%s] 没有指定数据类型或者Client=0，忽略该列 \033[0m" % (i,coltitle)
				continue

			colnames.append(colname)
			coltypes.append(coltype)
			coltitles.append(coltitle)

			coltype = dataType2SqlType(coltype_row[i].value.strip())
			if coltype == None:
				print u"\033[0;31m [%s]列 存在没有制定的数据类型[%s]，默认是 TEXT \033[0m" % (i,coltype_row[i].value)
				coltype = "text"
			words[colname] = coltype

		#print "Words: ",words
		datas = []
		for j in range(6,sheet.nrows):
			data_row = sheet.row(j)
			row = []
			for k in range(0,len(data_row)):
				if coltype_row[k].value.strip() == "" or ingor_row[k].value == 0:
					#print u"\033[0;31m [%s]列 列名[%s] 没有指定数据类型或者Client=0，忽略该列 \033[0m" % (i,coltitle)
					continue
				try:
					valueTmp = data_row[k].value
					if isUnicode(valueTmp):
						valueTmp = valueTmp.encode('utf-8')
					
					valueTmp = data2ValueWithType(coltype_row[k].value.strip(),valueTmp)
					valueTmp = valueTmp.decode('utf-8')
					if valueTmp == None:
						print u"\033[0;31m 解析单元格错误 [%s] 在表 [%s]  行 [%s] 列 [%s] \033[0m" % ("unknow type",name,j,colnames[k])
					else:
						row.append(valueTmp)
				except Exception, e:
					print u"\033[0;31m 解析单元格错误 [%s] 在表 [%s]  行 [%s] 列 [%s] \033[0m" % ("unknow type",name,j,colnames[k])
					print u"\033[0;31m 原因:%s \033[0m" %(e)
			
			datas.append(row)
		tables[objname_row[0].value] = {"words" : words, "colnames" : colnames, "coltypes" : coltypes, "coltitles" : coltitles,"datas" : datas}

	except ValueError as e:
		print(u"\033[0;31m 表不合规 跳过表 [%s] \033[0m" % (name))
		print u"\033[0;31m 原因:%s \033[0m" %(e)
		print ""
		return
	print u"\033[0;32m 表 [%s] 导出成功 \033[0m" %(name)
	print ""

def export2Table():
	global tables
	tmpl = "tables.tpl"
	output = "output/tables.sql"

	env = Environment(loader=PackageLoader('xls2db','template'))
	template = env.get_template(tmpl)
	outputData = template.render(tables = tables)
	filePath = os.path.join(baseDir,output)
	print "TableFilePath:",filePath
	f = codecs.open(filePath,'w','utf-8')
	f.write(outputData)
	f.close()

def export2SqlData():
	global tables
	tmpl = "sqlDatas.tpl"
	env = Environment(loader=PackageLoader('xls2db','template'))
	template = env.get_template(tmpl)

	for key,value in tables.iteritems():
		print "Key: " ,key
		output = "output/%s.sql" % key
		print "Output:",output
		outputData = template.render(tableName=key,words=value["colnames"],datas=value["datas"])
		filePath = os.path.join(baseDir,output)
		print "SqlDataFilePath: [%s] : %s" % (key,filePath)
		f = codecs.open(filePath,'w','utf-8')
		f.write(outputData)
		f.close()

def export2CSharp():
	global tables
	tmpl = "cSharp.tpl"
	env = Environment(loader=PackageLoader('xls2db','template'))
	template = env.get_template(tmpl)
	output = "output/SqlDataStruct.cs"
	outputData = template.render(tables=tables)
	filePath = os.path.join(baseDir,output)
	print "CSharpFilePath: %s" % (filePath)
	f = codecs.open(filePath,'w','utf-8')
	f.write(outputData)
	f.close()

parser = OptionParser(usage='"usage:%prog [options]"',version="%prog 1.0")
parser.add_option('-i', '--input', dest='input', help=u'-i 输入excel配置文件', action='store')

if __name__ == '__main__':
	options, args = parser.parse_args()
	print 'Options:', options
	print 'Args:', args
	print u"\033[0;32m xlc表有点大，等待一会... \033[0m"
	baseDir = os.path.split(os.path.realpath(__file__))[0]
	wb = xlrd.open_workbook(os.path.join(baseDir,options.input))
	sheets = wb.sheets()

	tables = collections.OrderedDict()
	for i in range(0,len(sheets)):
		sheet = sheets[i]
		if sheet.name == u"语言包":
			print u"\033[0;32m  检测到语言包，跳过，不倒入 \033[0m"
			continue
		if sheet.name == "MARK":
			print u"\033[0;32m 检测到MARK表，停止导入后面的表 \033[0m"
			break
		convertSheet(sheet,baseDir)

	# print 'Tables:',tables
	export2Table()
	export2SqlData()
	export2CSharp()

为了满足策划的 “一键生成”策略，可以用pipy或者shell编写一个可执行文件，如果在Mac上可以直接用Python。

Shell编程扫盲： C语言中文网

oneKey.sh

#! /bin/sh

#####################################
#									#
# Usage: sh oneKey.sh XXX配置.xlsx 	#
#									#
#####################################
cd "$(dirname "$0")"
pwd
config_file=$1
echo $config_file
if [[ ${config_file} == "" ]]; then
	echo "===== Please Input Config Excel File ====="
	read config_file
fi

#"获取Excel 文件名"
tmp=${config_file##*/}
tmp=${tmp%.*}

echo "==> 执行Python 解析 "$tmp".Excel"
python xls2db.py -i ${config_file}
echo "==> 生成  "$tmp".Sql文件"

#执行.sql文件
BasePath=$(cd "$(dirname "$0")";pwd)
dbPath=${BasePath}"/db"
sqlPath=${BasePath}"/output"
dbFile=${dbPath}/data.db
sqlFile=${sqlPath}/$tmp.sql

echo "==> 重写/db/"$tmp".sql"
if [[ -f $sqlFile ]]; then
	cp $sqlFile $dbPath
fi

echo "Excute "$tmp".sql"
cd $dbPath

sqlite3 data.db "delete from "$tmp | echo "删除 "$tmp" 完成!"
sqlite3 data.db ".read "$tmp".sql" | echo "写入 "$tmp" 完成!"

echo "All Done!"