内网 大数据同步至外网(云服务器) 一
描述：内网数据库 多表大量数据 需要同步至外网。
思路方案：1.window定时任务，每天凌晨1点 (多线程或者单线程) 同步前一天的数据。
2.因可能会出现同步失败的 情况,以及会设置到修改数据，内网数据库各表要有 创建时间和修改时间，外网结构 也要有创建时间和修改时间，方便数据核对，同时 批次插入或修改数据库时，若失败 因邮件或日志文件通知开发人员。
3.内网数据库 多张表，需要封装一个公共的获取 数据集合的方法(此方法 分页获取表 数据，使用类反射 调用类里面的方法)
4.内网数据 同步到外网，用http协议发送 post 数据 请求，外网(云服务器)上 要做安全验证 (仅处理指定ip地址的 请求)
5.外网(云服务器)上 数据批量保存，使用 SqlBulkCopy，封装 一个公共的批量保存方法。(若是修改数据 则先删除原来的数据再新增)。
核心代码：
a.实体对象，继承自basePush
public class basePush
{
///
/// 数据库目标表数据列名 对应关系
///
/// DictionaryColumn()
{
Dictionary ls = new Dictionary();
return ls;
}
}
[Serializable]
public class DiseasebuweiInfo : basePush
{
public string disname { get; set; }
public string bw { get; set; }
public int cnt { get; set; }
public Int64 rn { get; set; }
public DateTime? CreateTime { get; set; }
public DateTime? UpdateTime { get; set; }
} b.任务方法
///
///
/// 每天 凌晨1点后执行
///
private static void TaskRunDiseasebuwei()
{
CHC.DAL.Log.Loger.Log("开始->同步Diseasebuwei：TaskRunDiseasebuwei", "系统任务");
GetPostList("DiseasebuweiInfo", "update"); //修改
}
private static void TaskRunSysdisease()
{
CHC.DAL.Log.Loger.Log("开始->同步SysdiseaseInfo：TaskRunSysdisease", "系统任务");
GetPostList("SysdiseaseInfo", "create");//新增
} c.获取数据集合的统一方法
//List对象 post数据
private static void GetPostList(string name, string typeName)
{
int page = 1;
int rows = 5000;
int total = 0;
int pageCount = 1; //数据库 取数据源 ,每次同步1000个
string strName = typeof(T).Name;
try
{
//DateTime datetime = DateTime.Now;
//前一天的
string DayOfDay = LastRunDate.ToString("yyyy-MM-dd HH:00:00");
while (page <= pageCount)
{
Type typ = typeof(TaskPushDAL);
MethodInfo methstr = typ.GetMethod(string.Format("Get{0}List", name));
object[] objPara = new object[] { DayOfDay, (page - 1) * rows + 1, page * rows, total, typeName };
List infoList = methstr.Invoke(null, objPara) as List;
total = (int)objPara[3];
//List infoList = TaskPushDAL.GetDiseasebuweiInfoList(DayOfDay, (page - 1) * rows + 1, page * rows, out total);
if (page == 1)
{
pageCount = (total / rows) + (total % rows != 0 ? 1 : 0);
}
if (infoList != null && infoList.Count > 0)
{
//post 数据
string result = string.Empty;
result = ResponseTask(infoList, name, typeName);
//通知成功
if (result.Contains("ok"))
{
CHC.DAL.Log.Loger.Log(string.Format("{0}通知成功!===result:{1}", strName, result));
}
else
{
CHC.DAL.Log.Loger.Log(string.Format("{0}通知失败!===result:{1},当前页数:{2},操作类型:{3}",
strName, result, page, typeName));
}
}
page = page + 1;
}
}
catch (Exception ex)
{
CHC.DAL.Log.Loger.Error(ex, string.Format("同步{0}异常", strName));
}
}
//访问数据库 分页获取数据
public static List GetDiseasebuweiInfoList(string DayOfDay, int pageStart, int pageEnd, out int total,string typename)
{
StringBuilder sql = new StringBuilder();
sql.AppendFormat(@"select COUNT(1) from [tb_diseasebuwei] a where {1}>'{0}' ", DayOfDay,
typename == "update" ? "a.UpdateTime" : "a.CreateTime");
total = SqlUtility.ExecuteScalar(_connectionBig, sql.ToString(), null);
string s = string.Format(@"select * from (select a.disname,a.bw,a.cnt,a.rn,dn=ROW_NUMBER()OVER(ORDER BY a.disname) from
[tb_diseasebuwei] a with(nolock) where {3}>'{0}' )tmp where dn between {1} and {2} "
, DayOfDay, pageStart, pageEnd, typename == "update" ? "a.UpdateTime" : "a.CreateTime");
return SqlUtility.ExecuteObjectList(_connectionBig, s, null);
}
//修改时，先删除旧数据
public static string DeleteDiseasebuweiInfo(List list)
{
string strs = string.Join("','", list.Select(v => v.disname));
string sql = string.Format("delete from tb_diseasebuwei where disname in('{0}')", strs);
return sql;
} d.post数据
//发送 post 数据 请求
private static string ResponseTask(List list, string Name, string typeName)
{
string result = string.Empty;
try
{
string serverURL = string.Empty;
if (ConfigurationManager.AppSettings["serverURL"] != null)
{
serverURL = ConfigurationManager.AppSettings["serverURL"];
serverURL = serverURL + NotifyUrl;
}
if (list != null && list.Count > 0 && !string.IsNullOrEmpty(serverURL) && !string.IsNullOrEmpty(Name))
{
string ModelList = Newtonsoft.Json.JsonConvert.SerializeObject(list,
new Newtonsoft.Json.Converters.IsoDateTimeConverter() { DateTimeFormat = "yyyy-MM-dd HH:mm:ss" });
TaskData dataInfo = new TaskData() { MothodName = Name, ModelList = ModelList, ModeType = typeName };
string param = Newtonsoft.Json.JsonConvert.SerializeObject(dataInfo);
result = GetHTMLByPost(serverURL, param).ToLower();
}
else
{
result = "参数不能为空";
}
}
catch (Exception ex)
{
LogUtil.LogError("发送 post 数据 请求异常", ex);
result = ex.Message;
}
return result;
}
public static string GetHTMLByPost(string Url, string Params)
{
string result = "";
string encoding = "utf-8";
byte[] bytes = Encoding.GetEncoding(encoding).GetBytes(Params);
HttpWebRequest httpWebRequest = (HttpWebRequest)WebRequest.Create(Url);
httpWebRequest.Method = "POST";
httpWebRequest.ContentType = "application/x-www-form-urlencoded";
httpWebRequest.ContentLength = (long)bytes.Length;
using (Stream requestStream = httpWebRequest.GetRequestStream())
{
requestStream.Write(bytes, 0, bytes.Length);
}
using (WebResponse response = httpWebRequest.GetResponse())
{
StreamReader streamReader = new StreamReader(response.GetResponseStream(), Encoding.UTF8);
result = streamReader.ReadToEnd();
response.Close();
}
return result;
} e.云服务器 接收数据
public class PageTask : PageAjaxHandler
{
private string checkIp = "127.0.0.1";//指定的ip
public PageTask(HttpContext context) : base(context) {
if (ConfigurationManager.AppSettings["checkIp"] != null)
{
checkIp = ConfigurationManager.AppSettings["checkIp"];
}
}
public void SetData()
{
ResponseData responseData = new ResponseData();
CHC.DAL.Log.Loger.Log("大数据保存-----");
try
{
//获取 请求的ip地址
string requestIp = Lin.ToolKit.Common.RequestHelper.GetIPAddress();
if (!string.IsNullOrEmpty(requestIp) && checkIp.Contains(requestIp))
{
string jasonstrs = new System.IO.StreamReader(Request.InputStream).ReadToEnd();
TaskData info = Newtonsoft.Json.JsonConvert.DeserializeObject(jasonstrs);
//UrlParam url = new UrlParam(Request);
//string MothodName = url.Querys["MothodName"];//方法
string MothodName = info.MothodName;
string ModelList = info.ModelList;//list 对象 json格式
string ModeType = info.ModeType;
string result = string.Empty;
//list对象保存到数据库
switch (MothodName)
{
case "DiseasebuweiInfo":
List list = Newtonsoft.Json.JsonConvert.DeserializeObject>(ModelList);
result = TaskPushBLL.SaveDataPush(MothodName, ModeType, list, "tb_diseasebuwei");
break;
case "SysdiseaseInfo":
List SysdiseaseInfoList = Newtonsoft.Json.JsonConvert.DeserializeObject>(ModelList);
result = TaskPushBLL.SaveDataPush(MothodName, ModeType, SysdiseaseInfoList, "tb_sysdisease");
break;
}
if (string.IsNullOrEmpty(result))
{
responseData.ret = "ok";
}
else
{
responseData.msg = result;
}
}
else
{
responseData.ret = "error";
responseData.msg = "请求的ip地址异常";
}
}
catch (Exception ex)
{
responseData.msg = ex.Message;
}
WriteObject(responseData);
}
} f.大数据保存
#region 大数据保存
///
/// 保存数据
///
/// 实体对象
/// 需要反射的方法名
/// 新增或修改
/// 实体对象集合
/// 目标表名
/// (string name, string typename, List list, string tableName)
{
string str = string.Empty;
string strName = typeof(T).Name;
if (list != null && list.Count > 0)
{
try
{
Dictionary DictionaryColumn = new Dictionary();
MethodInfo methstr = typeof(T).GetMethod("DictionaryColumn");
if (methstr != null)
{
DictionaryColumn = methstr.Invoke(null, null) as Dictionary;
}
//TaskPushDAL.Sqlbulkcopy(list, tableName, DictionaryColumn);
str = TaskPushDAL.Sqlbulkcopy(name, typename, list, tableName, DictionaryColumn);
CHC.DAL.Log.Loger.Log(string.Format("保存({0}){1}数据共{2}条!===result:{3}", typename, strName, list.Count, str));
}
catch (Exception ex)
{
str = ex.Message;
CHC.DAL.Log.Loger.Error(ex);
}
}
return str;
}
#endregion g.使用 SqlBulkCopy 批量处理数据
///
/// typeName 为update :先删除旧数据 再保存新数据 ，为create:直接插入新数据
///
/// (string name, string typeName, List data, string tableName, Dictionary DictionaryColumn)
{
string str = string.Empty;
List pList = new List();//创建属性的集合
DataTable dt = new DataTable();
//把所有的public属性加入到集合 并添加DataTable的列
Array.ForEach(typeof(T).GetProperties(), p =>
{
pList.Add(p);
Type colType = p.PropertyType;
if ((colType.IsGenericType) && (colType.GetGenericTypeDefinition() == typeof(Nullable<>)))
{
colType = colType.GetGenericArguments()[0];
}
dt.Columns.Add(new DataColumn(p.Name, colType));
});
foreach (var item in data)
{
DataRow row = dt.NewRow();
//pList.ForEach(p => row[p.Name] = (item == null ? null : p.GetValue(item, null)));
foreach (var p in pList)
{
object tvalue = p.GetValue(item, null);
row[p.Name] = tvalue == null ? DBNull.Value : tvalue;
}
dt.Rows.Add(row);
}
string strName = typeof(T).Name;
using (SqlConnection conn = new SqlConnection(_connection))
{
conn.Open();
SqlTransaction sqlbulkTransaction = conn.BeginTransaction();
#region 处理批量插入对象
SqlBulkCopy bulk = new SqlBulkCopy(conn, SqlBulkCopyOptions.CheckConstraints, sqlbulkTransaction)
{
DestinationTableName = tableName, /*设置数据库目标表名称*/
BatchSize = dt.Rows.Count, /*每一批次中的行数*/
};
bulk.BulkCopyTimeout = 5000; //指定操作完成的Timeout时间
if (DictionaryColumn != null && DictionaryColumn.Count > 0)
{
foreach (var item in DictionaryColumn)
{
bulk.ColumnMappings.Add(item.Key, item.Value);//不一致 先指定对应关系 ；源列：目标列
}
}
else
{
pList.ForEach(p =>
{
bulk.ColumnMappings.Add(p.Name, p.Name);//列名和 对象属性一致
});
}
#endregion
try
{
if (typeName == "update") //先删除 再新增
{
Type typ = typeof(TaskPushDAL);
MethodInfo methstr = typ.GetMethod(string.Format("Delete{0}", name));
object[] objPara = new object[] { data };
string sql = methstr.Invoke(null, objPara).ToString();
SqlCommand sqlComm = new SqlCommand(sql, conn, sqlbulkTransaction);
sqlComm.CommandTimeout = 300;
int count=sqlComm.ExecuteNonQuery();
CHC.DAL.Log.Loger.Log(string.Format("Sqlbulkcopy,{0},Delete的数量:{1}", strName, count));
}
bulk.WriteToServer(dt);//写入表
CHC.DAL.Log.Loger.Log(string.Format("Sqlbulkcopy,{0},批量执行数量:{1}", strName, dt.Rows.Count));
sqlbulkTransaction.Commit();
}
catch (Exception ex)
{
str = ex.Message;
sqlbulkTransaction.Rollback();
CHC.DAL.Log.Loger.Error(ex);
}
finally
{
bulk.Close();
conn.Close();
}
}
return str;
}