作者博客迁移至博客园：http://www.cnblogs.com/xiaodf/
package iie.hadoop.hcatalog.spark;
import iie.udps.common.hcatalog.SerHCatInputFormat;
import iie.udps.common.hcatalog.SerHCatOutputFormat;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import org.apache.hive.hcatalog.common.HCatUtil;
import org.apache.hive.hcatalog.data.DefaultHCatRecord;
import org.apache.hive.hcatalog.data.HCatRecord;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
import org.apache.spark.Accumulator;
import org.apache.spark.SerializableWritable;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.io.RCFileOutputFormat;
import org.apache.hadoop.hive.serde.serdeConstants;
//import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hive.hcatalog.mapreduce.OutputJobInfo;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.api.java.function.PairFunction;
import org.apache.spark.broadcast.Broadcast;
import org.apache.thrift.TException;
import scala.Tuple2;
/**
* spark+hcatalog 实现表的复制功能， 并将原表一列数据变成大写存到新表 ; create table test(name String,age
* int); 执行命令：spark-submit --master yarn-cluster --class
* iie.hadoop.hcatalog.spark.LowerUpperCaseConvert /home/xdf/test.jar -c
* /user/xdf/stdin.xml
*
* @author xiaodongfang
*
*/
public class LowerUpperCaseConvert {
private static Accumulator inputDataCount;
private static Accumulator outputDataCount;
@SuppressWarnings("rawtypes")
public static void main(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: <-c> ");
System.exit(1);
}
String stdinXml = args[1];
String userName = null;
String jobinstanceid = null;
String operatorName = null;
String dbName = null;
String inputTabName = null;
String operFieldName = null;
int fieldCount = 0;
// 读取stdin.xml文件
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
FSDataInputStream dis = fs.open(new Path(stdinXml));
InputStreamReader isr = new InputStreamReader(dis, "utf-8");
BufferedReader read = new BufferedReader(isr);
String tempString = "";
String xmlParams = "";
while ((tempString = read.readLine()) != null) {
xmlParams += "\n" + tempString;
}
read.close();
xmlParams = xmlParams.substring(1);
// 获取xml文件中的参数值
OperatorParamXml operXML = new OperatorParamXml();
List list = operXML.parseStdinXml(xmlParams);
userName = list.get(0).get("userName").toString();
dbName = list.get(0).get("dbName").toString();
inputTabName = list.get(0).get("inputTabName").toString();
operatorName = list.get(0).get("operatorName").toString();
jobinstanceid = list.get(0).get("jobinstanceid").toString();
fieldCount = Integer.parseInt(list.get(0).get("fieldCount").toString());
// 设置输出表字段名及类型
ArrayList fieldName = new ArrayList();
ArrayList fieldType = new ArrayList();
for (int i = 1; i <= fieldCount; i++) {
fieldName.add(list.get(0).get("fieldName" + i).toString());
fieldType.add(list.get(0).get("fieldType" + i).toString());
}
String[] fieldNames = new String[fieldCount];
String[] fieldTypes = new String[fieldCount];
// 设置输出表的名字
String outputTable = "tmp_" + UUID.randomUUID().toString().replace('-', '_');
// 获取表字段名字和类型
for (int j = 0; j < fieldCount; j++) {
fieldNames[j] = fieldName.get(j);
fieldTypes[j] = fieldType.get(j);
System.out.println("====fieldName=====" + fieldNames[j]);
System.out.println("====fieldType=====" + fieldTypes[j]);
}
System.out.println("====fieldCount=====" + fieldCount);
// 创建hive表
HCatSchema schema = getHCatSchema(dbName, inputTabName);
createTable(dbName, outputTable, schema);
// 将输入表字段数据转换为大写，写入输出表文件中
JavaSparkContext jsc = new JavaSparkContext(
new SparkConf().setAppName("LowerUpperCaseConvert"));
inputDataCount = jsc.accumulator(0);
outputDataCount = jsc.accumulator(0);
// 要操作的字段名称及字段序号
operFieldName = fieldNames[0];
System.out.println("====operFieldName======" + operFieldName);
int position = schema.getPosition(operFieldName);
JavaRDD> rdd1 = LowerUpperCaseConvert
.lowerUpperCaseConvert(jsc, dbName, inputTabName, position);
LowerUpperCaseConvert.storeToTable(rdd1, dbName, outputTable);
jsc.stop();
// 设置输出xml文件参数
List listOut = new ArrayList();
Map mapOut = new HashMap();
mapOut.put("jobinstanceid", jobinstanceid);
mapOut.put("dbName", dbName);
mapOut.put("outputTable", outputTable);
mapOut.put("inputDataCount", inputDataCount.value().toString());
mapOut.put("outputDataCount", outputDataCount.value().toString());
String operFieldType = fieldTypes[0];// 要操作的字段类型
if (operFieldType.equalsIgnoreCase("String")) {
// 创建正常输出xml文件
listOut.add(mapOut);
String hdfsOutXml = "/user/" + userName + "/optasks/"
+ jobinstanceid + "/" + operatorName + "/out"
+ "/stdout.xml";
operXML.genStdoutXml(hdfsOutXml, listOut);
} else {
// 创建错误输出xml文件
String errorMessage = "fieldType is not string!!!";
String errotCode = "80001";
mapOut.put("errorMessage", errorMessage);
mapOut.put("errotCode", errotCode);
listOut.add(mapOut);
String hdfsErrorXml = "/user/" + userName + "/optasks/"
+ jobinstanceid + "/" + operatorName + "/out"
+ "/stderr.xml";
operXML.genStderrXml(hdfsErrorXml, listOut);
}
System.exit(0);
}
@SuppressWarnings("rawtypes")
public static JavaRDD> lowerUpperCaseConvert(
JavaSparkContext jsc, String dbName, String inputTabName,
int position) throws IOException {
Configuration inputConf = new Configuration();
SerHCatInputFormat.setInput(inputConf, dbName, inputTabName);
JavaPairRDD rdd = jsc
.newAPIHadoopRDD(inputConf, SerHCatInputFormat.class,
WritableComparable.class, SerializableWritable.class);
final Broadcast posBc = jsc.broadcast(position);
// 获取表记录集
JavaRDD> result = null;
final Accumulator output = jsc.accumulator(0);
final Accumulator input = jsc.accumulator(0);
result = rdd
.map(new Function, SerializableWritable>() {
private static final long serialVersionUID = -2362812254158054659L;
private final int postion = posBc.getValue().intValue();
public SerializableWritable call(
Tuple2 v)
throws Exception {
HCatRecord record = (HCatRecord) v._2.value();
// +1 inport
input.add(1);
List newRecord = new ArrayList(record
.size());
for (int i = 0; i < record.size(); ++i) {
newRecord.add(record.get(i));
}
/*
* if (ok) +1 outport1 else +1 errport
*/
newRecord.set(postion, newRecord.get(postion)
.toString().toUpperCase());
output.add(1);
return new SerializableWritable(
new DefaultHCatRecord(newRecord));// 返回记录
}
});
inputDataCount = input;
outputDataCount = output;
return result;
}
@SuppressWarnings("rawtypes")
public static void storeToTable(
JavaRDD> rdd, String dbName,
String tblName) {
Job outputJob = null;
try {
outputJob = Job.getInstance();
outputJob.setJobName("lowerUpperCaseConvert");
outputJob.setOutputFormatClass(SerHCatOutputFormat.class);
outputJob.setOutputKeyClass(WritableComparable.class);
outputJob.setOutputValueClass(SerializableWritable.class);
SerHCatOutputFormat.setOutput(outputJob,
OutputJobInfo.create(dbName, tblName, null));
HCatSchema schema = SerHCatOutputFormat.getTableSchema(outputJob
.getConfiguration());
SerHCatOutputFormat.setSchema(outputJob, schema);
} catch (IOException e) {
e.printStackTrace();
}
// 将RDD存储到目标表中
rdd.mapToPair(
new PairFunction, WritableComparable, SerializableWritable>() {
private static final long serialVersionUID = -4658431554556766962L;
@Override
public Tuple2> call(
SerializableWritable record)
throws Exception {
return new Tuple2>(
NullWritable.get(), record);
}
}).saveAsNewAPIHadoopDataset(outputJob.getConfiguration());
}
// 创建表结构
public static void createTable(String dbName, String tblName,
HCatSchema schema) {
HiveMetaStoreClient client = null;
try {
HiveConf hiveConf = HCatUtil.getHiveConf(new Configuration());
try {
client = HCatUtil.getHiveClient(hiveConf);
} catch (MetaException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} catch (IOException e) {
e.printStackTrace();
}
try {
if (client.tableExists(dbName, tblName)) {
client.dropTable(dbName, tblName);
}
} catch (TException e) {
e.printStackTrace();
}
List fields = HCatUtil.getFieldSchemaList(schema
.getFields());
System.out.println(fields);
Table table = new Table();
table.setDbName(dbName);
table.setTableName(tblName);
StorageDescriptor sd = new StorageDescriptor();
sd.setCols(fields);
table.setSd(sd);
sd.setInputFormat(RCFileInputFormat.class.getName());
sd.setOutputFormat(RCFileOutputFormat.class.getName());
sd.setParameters(new HashMap());
sd.setSerdeInfo(new SerDeInfo());
sd.getSerdeInfo().setName(table.getTableName());
sd.getSerdeInfo().setParameters(new HashMap());
sd.getSerdeInfo().getParameters()
.put(serdeConstants.SERIALIZATION_FORMAT, "1");
sd.getSerdeInfo().setSerializationLib(
org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe.class
.getName());
Map tableParams = new HashMap();
table.setParameters(tableParams);
try {
client.createTable(table);
System.out.println("Create table successfully!");
} catch (TException e) {
e.printStackTrace();
return;
} finally {
client.close();
}
}
// 获得HCatSchema
public static HCatSchema getHCatSchema(String dbName, String tblName) {
Job outputJob = null;
HCatSchema schema = null;
try {
outputJob = Job.getInstance();
outputJob.setJobName("getHCatSchema");
outputJob.setOutputFormatClass(SerHCatOutputFormat.class);
outputJob.setOutputKeyClass(WritableComparable.class);
outputJob.setOutputValueClass(SerializableWritable.class);
SerHCatOutputFormat.setOutput(outputJob,
OutputJobInfo.create(dbName, tblName, null));
schema = SerHCatOutputFormat.getTableSchema(outputJob
.getConfiguration());
} catch (IOException e) {
e.printStackTrace();
}
return schema;
}
}
