Hadoop基础（七） --- hadoop的串行化和反串行化，BytesWritable，NullWritable，ObjectWritable，MapWri
一、hadoop的串行化和反串行化
----------------------------------------------------
@Test
public void ts01() throws Exception
{
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream oos = new DataOutputStream(baos);
IntWritable intw = new IntWritable(255);
intw.write(oos);
oos.close();
baos.close();
byte [] buf = baos.toByteArray();
System.out.println(buf.length);
System.out.println("-----------------------");
intw.readFields(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
System.out.println(intw.get());
}
@Test
public void ts02() throws Exception
{
Person p = new Person();
p.setName(new Text("toms"));
p.setAge(new IntWritable(12));
p.setMale(new BooleanWritable(true));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream bos = new DataOutputStream(baos);
p.write(bos);
byte [] buf = baos.toByteArray();
bos.close();
baos.close();
System.out.println(buf.length);
Person p1 = new Person();
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf));
p1.readFields(dis);
System.out.println(p1.getName());
System.out.println(p1.getAge());
System.out.println(p1.getMale());
}
@Test
public void ts03() throws Exception
{
Person p = new Person();
Address ad = new Address();
p.setName(new Text("toms"));
p.setAge(new IntWritable(12));
p.setMale(new BooleanWritable(true));
ad.setNation(new Text("china"));
ad.setCity(new Text("beijing"));
ad.setVillage(new Text("zxz"));
p.setAddr(ad);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream bos = new DataOutputStream(baos);
p.write(bos);
byte [] buf = baos.toByteArray();
bos.close();
baos.close();
System.out.println(buf.length);
Person p1 = new Person();
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(buf));
p1.readFields(dis);
System.out.println(p1.getName());
System.out.println(p1.getAge());
System.out.println(p1.getMale());
System.out.println(p1.getAddr().getNation());
System.out.println(p1.getAddr().getCity());
System.out.println(p1.getAddr().getVillage());
}
二、 BytesWritable
-------------------------
1.写入的时候，先写数组长度，在写数组数据
2.write(size);
3.write(bytes,0,size);
三、NullWritable
----------------------------
1.没有数据的单例writable对象
2.单例模式中的饿汉式，使用NullWritable.get()获取
3.充当占位符
四、ObjectWritable
--------------------------------
/**
* 测试 ObjectWritable
* @throws Exception
*/
@Test
public void ts04() throws Exception
{
String [] strs = {"aaa","ccc","vvv"};
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream oos = new DataOutputStream(baos);
ObjectWritable obj = new ObjectWritable(strs);
obj.write(oos);
oos.close();
baos.close();
byte [] buf = baos.toByteArray();
System.out.println(buf.length);
System.out.println("-----------------------");
ObjectWritable obj2 = new ObjectWritable();
obj2.readFields(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
String[] st = (String[])obj2.get();
System.out.println(st[0]);
}
五、MapWritable
-------------------------------
1.使用MapWritable类传输自定义Writable(Person)实现类，正确完成序列化和反序列化的过程
/**
* 测试MapWritable
* @throws Exception
*/
@Test
public void ts05() throws Exception
{
MapWritable map = new MapWritable();
map.put(new IntWritable(100), new Text("tom1"));
map.put(new IntWritable(200), new Text("tom2"));
map.put(new IntWritable(300), new Text("tom3"));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream oos = new DataOutputStream(baos);
map.write(oos);
MapWritable map1 = new MapWritable();
map1.readFields(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
Text tx = (Text)map1.get(new IntWritable(100));
System.out.println(tx);
}
/**
* 测试MapWritable
* @throws Exception
*/
@Test
public void ts06() throws Exception
{
MapWritable map = new MapWritable();
map.put(new IntWritable(100), new Person(new Text("tom1"),new IntWritable(1),new BooleanWritable(true),new Address()));
map.put(new IntWritable(200), new Person(new Text("tom2"),new IntWritable(2),new BooleanWritable(true),new Address()));
map.put(new IntWritable(300), new Person(new Text("tom3"),new IntWritable(3),new BooleanWritable(true),new Address()));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream oos = new DataOutputStream(baos);
map.write(oos);
MapWritable map1 = new MapWritable();
map1.readFields(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
Person tx = (Person)map1.get(new IntWritable(200));
System.out.println(tx.getName());
}
/**
* 测试MapWritable
* @throws Exception
*/
@Test
public void ts07() throws Exception
{
MapWritable map = new MapWritable();
map.put(new Person(new Text("tom1"),new IntWritable(1),new BooleanWritable(true),new Address()),new Text("222"));
map.put(new Person(new Text("tom2"),new IntWritable(2),new BooleanWritable(true),new Address()),new Text("333"));
map.put(new Person(new Text("tom3"),new IntWritable(3),new BooleanWritable(true),new Address()),new Text("444"));
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream oos = new DataOutputStream(baos);
map.write(oos);
MapWritable map1 = new MapWritable();
map1.readFields(new DataInputStream(new ByteArrayInputStream(baos.toByteArray())));
Text tx = (Text)map1.get(new Person(new Text("tom3"),new IntWritable(3),new BooleanWritable(true),new Address()));
System.out.println(tx);
}-------------------------------------------------------------------------
下面是自定义的Person类和Address类，注意，这两个类因为使用到hashMap机制，所以要自动生成 hashCode方法！！
-------------------------------------------------------------------------
public class Address implements Writable{
private Text nation = new Text();
private Text city = new Text();
private Text village = new Text();
@Override
public void write(DataOutput out) throws IOException {
nation.write(out);
city.write(out);
village.write(out);
}
@Override
public void readFields(DataInput in) throws IOException {
nation.readFields(in);
city.readFields(in);
village.readFields(in);
}
//....
/*此处省略 构造函数，get ，set , hashcode()*/
//....
}
public class Person implements Writable , Comparable {
private Text name = new Text(); //hadoop 字符串类型
private IntWritable age = new IntWritable(); //hadoop int 类型
private BooleanWritable male = new BooleanWritable(); //hadoop bool 类型
private Address addr = new Address();
/**
* 序列化
*/
@Override
public void write(DataOutput out) throws IOException {
name.write(out);
age.write(out);
male.write(out);
addr.getNation().write(out);
addr.getCity().write(out);
addr.getVillage().write(out);
}
/**
* 反序列化
*/
@Override
public void readFields(DataInput in) throws IOException {
name = new Text();
age = new IntWritable();
male = new BooleanWritable();
name.readFields(in);
age.readFields(in);
male.readFields(in);
addr = new Address();
addr.getNation().readFields(in);
addr.getCity().readFields(in);
addr.getVillage().readFields(in);
}
//....
/*此处省略 构造函数，get ，set , hashcode()*/
//....
}
六、Hadoop为什么不使用java的串行化和RMI？
--------------------------------------
1.Hadoop要求的是高效，高性能的进程间通信，要支持互操作，而java串行化太复杂，达不到至精至简的效果
2.Hadoop需要精确的控制连接，延迟和缓冲的处理方式，RMI没有