spark序列化和反序列化对象的实现,java版本

1.定义一个项目

mvn archetype:generate -DgroupId=com.packt.samples -DartifactId=objectstream -Dversion=1.0.0 -DinteractiveMode=false -DarchetypeCatalog=internal

2.修改pom文件如下

[root@localhost objectstream]# cat pom.xml

  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">

  4.0.0

  com.packt.samples

  objectstream

  jar

  1.0.0

  objectstream

  http://maven.apache.org

 

   

      junit

      junit

      3.8.1

      test

   

 

 

       

        org.apache.maven.plugins   

        maven-compiler-plugin   

           

            1.8   

            1.8   

           

     

   

      org.apache.maven.plugins

      maven-assembly-plugin

      2.4

     

       

          jar-with-dependencies

       

       

         

            true

            com.packt.samples.App

         

       

     

     

       

          assemble-all

          package

         

            single

         

       

     

   

 

[root@localhost objectstream]#

3.入口类是 App

[root@localhost objectstream]# cat src/main/java/com/packt/samples/App.java

package com.packt.samples;

import java.nio.ByteBuffer;

/**

* Hello world!

*

*/

public class App

{

    public static void main( String[] args )

    {

        JavaSerializer ser = new JavaSerializer();

        SerializerInstance jsi = ser.newInstance();

        Task ti = new Task(1);

        ByteBuffer buf = jsi.serialize(ti);

        Task dd = (Task) jsi.deserialize(buf,Thread.currentThread().getContextClassLoader());

        System.out.println(dd);

    }

}

4.定义序列化类 JavaSerializer

package com.packt.samples;

public class JavaSerializer {

  public JavaSerializer() {

  }

  public SerializerInstance newInstance() {

    return new JavaSerializerInstance(100,true,Thread.currentThread().getContextClassLoader());

  }

}

5.负责真正序列化和反序列化的JavaSerializerInstance类

[root@localhost objectstream]# cat src/main/java/com/packt/samples/JavaSerializerInstance.java

package com.packt.samples;

import java.nio.ByteBuffer;

import java.io.OutputStream;

import java.io.InputStream;

public class JavaSerializerInstance extends SerializerInstance {

  public int counterReset;

  public boolean extraDebugInfo;

  public ClassLoader defaultClassLoader;

  public JavaSerializerInstance(int counterReset,boolean extraDebugInfo,ClassLoader defaultClassLoader) {

    this.counterReset = counterReset;

    this.extraDebugInfo = extraDebugInfo;

    this.defaultClassLoader = defaultClassLoader;

  }

  @Override

  public ByteBuffer serialize(T t) {

    ByteBufferOutputStream bos = new ByteBufferOutputStream();

    SerializationStream out = serializeStream(bos);

    out.writeObject(t);

    out.close();

    return bos.toByteBuffer();

  }

  @Override

  public T deserialize(ByteBuffer bytes) {

    ByteBufferInputStream bis = new ByteBufferInputStream(bytes);

    DeserializationStream in = deserializeStream(bis);

    return (T) in.readObject();

  }

  @Override

  public T deserialize(ByteBuffer bytes, ClassLoader loader) {

    ByteBufferInputStream bis = new ByteBufferInputStream(bytes);

    DeserializationStream in = deserializeStream(bis, loader);

    return in.readObject();

  }

  @Override

  public SerializationStream serializeStream(OutputStream s) {

    return new JavaSerializationStream(s, counterReset, extraDebugInfo);

  }

  @Override

  public DeserializationStream deserializeStream(InputStream s) {

    return new JavaDeserializationStream(s, defaultClassLoader);

  }

  public DeserializationStream deserializeStream(InputStream s, ClassLoader loader) {

    return new JavaDeserializationStream(s, loader);

  }

}

6.读写相关的字节流对象

[root@localhost objectstream]# cat src/main/java/com/packt/samples/ByteBufferOutputStream.java

package com.packt.samples;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.nio.ByteBuffer;

class ByteBufferOutputStream extends ByteArrayOutputStream {

  public int capacity;

  public boolean closed = false;

  public ByteBufferOutputStream() {

    this(32);

  }

  public ByteBufferOutputStream(int capacity) {

    super(capacity);

  }

  public int getCount() {

    return count;

  }


  @Override

  public void write(int b) {

    super.write(b);

  }

  @Override

  public void write(byte[] b, int off, int len) {

    super.write(b, off, len);

  }

  @Override

  public void reset() {

    super.reset();

  }

  @Override

  public void close() {

    if (!closed) {

      try {

        super.close();

      } catch(IOException e) {

        System.out.println(e);

      }

      closed = true;

    }

  }

  public ByteBuffer toByteBuffer() {

    return ByteBuffer.wrap(buf, 0, count);

  }

}

[root@localhost objectstream]# cat src/main/java/com/packt/samples/ByteBufferInputStream.java

package com.packt.samples;

import java.nio.ByteBuffer;

import java.io.InputStream;

class ByteBufferInputStream extends InputStream {

  private ByteBuffer buffer;

  public ByteBufferInputStream(ByteBuffer buffer) {

    this.buffer = buffer;

  }

  @Override

  public int read() {

    if (buffer == null || buffer.remaining() == 0) {

      cleanUp();

      return -1;

    } else {

      return buffer.get() & 0xFF;

    }

  }

  @Override

  public int read(byte[] dest) {

    return read(dest, 0, dest.length);

  }

  @Override

  public int read(byte[] dest, int offset, int length) {

    if (buffer == null || buffer.remaining() == 0) {

      cleanUp();

      return -1;

    } else {

      int amountToGet = Math.min(buffer.remaining(), length);

      buffer.get(dest, offset, amountToGet);

      return amountToGet;

    }

  }

  @Override

  public long skip(long bytes) {

    if (buffer != null) {

      long amountToSkip =  Math.min(bytes, new Integer(buffer.remaining()).longValue());

      buffer.position(buffer.position() + new Long(amountToSkip).intValue());

      if (buffer.remaining() == 0) {

        cleanUp();

      }

      return amountToSkip;

    } else {

      return 0L;

    }

  }

  private void cleanUp() {

    if (buffer != null) {

      buffer = null;

    }

  }

}

6.对输入和输出流进行抽象的两个抽象类

[root@localhost objectstream]# cat src/main/java/com/packt/samples/DeserializationStream.java

package com.packt.samples;

import java.util.Iterator;

import java.io.Closeable;

import java.io.IOException;

import java.io.EOFException;

abstract class DeserializationStream implements Closeable {

  abstract public T readObject();

  public T readKey() {

    return readObject();

  }

  public T readValue() {

    return readObject();

  }

  @Override

  abstract public void close();

  public Iterator asIterator() {

    return new NextIterator() {

      @Override

      protected T getNext() {

        T t = null;

        t = readObject();

        return t;

      }

      @Override

      protected void close() {

        DeserializationStream.this.close();

      }

    };

  }

  public Iterator asKeyValueIterator() {

    return new NextIterator() {

      @Override

      protected ObjectPair getNext() {

        ObjectPair op = null;

        op = new ObjectPair(readKey(), readValue());

        return op;

      }

      @Override

      protected void close() {

        DeserializationStream.this.close();

      }

    };

  }

}

[root@localhost objectstream]# cat src/main/java/com/packt/samples/SerializationStream.java

package com.packt.samples;

import java.util.Iterator;

import java.io.Closeable;

abstract class SerializationStream implements Closeable {


  public abstract SerializationStream writeObject(T t);

  public SerializationStream writeKey(T key) {

    return writeObject(key);

  }


  public SerializationStream writeValue(T value) {

    return writeObject(value);

  }

  public abstract void flush();

  @Override

  public abstract void close();

  public SerializationStream writeAll(Iterator iter) {

    while (iter.hasNext()) {

      writeObject(iter.next());

    }

    return this;

  }

}

7.两个抽象类的实现

[root@localhost objectstream]# cat src/main/java/com/packt/samples/JavaDeserializationStream.java

package com.packt.samples;

import java.io.InputStream;

import java.io.ObjectInputStream;

import java.io.ObjectStreamClass;

import java.io.IOException;

import java.util.Map;

import java.util.HashMap;

public class JavaDeserializationStream extends DeserializationStream {

  public InputStream in;

  public ClassLoader loader;

  public ObjectInputStream objIn;

  public Map primitiveMappings;

  public JavaDeserializationStream(InputStream in, ClassLoader loader) {

    this.in = in;

    this.loader = loader;

    try {

      this.objIn = new ObjectInputStream(in) {

        @Override

        public Class resolveClass(ObjectStreamClass desc) {

          Class cls = null;

          try {

            cls = Class.forName(desc.getName(), false, loader);

          } catch(ClassNotFoundException e) {

            System.out.println(e);

          }

          return cls;

        }

      };

    } catch(IOException e) {

      System.out.println(e);

    }

    primitiveMappings = new HashMap();

    primitiveMappings.put("boolean", Boolean.class);

    primitiveMappings.put("byte", Byte.class);

    primitiveMappings.put("char", Character.class);

    primitiveMappings.put("short", Short.class);

    primitiveMappings.put("int", Integer.class);

    primitiveMappings.put("long", Long.class);

    primitiveMappings.put("float", Float.class);

    primitiveMappings.put("double", Double.class);

    primitiveMappings.put("void", Void.class);

  }

  public T readObject() {

    T t = null;

    try {

      t = (T) objIn.readObject();

    } catch(IOException e) {

      System.out.println(e);

    } catch(ClassNotFoundException e) {

      System.out.println(e);

    }

    return t;

  }


  public void close() {

    try {

      objIn.close();

    } catch(IOException e) {

      System.out.println(e);

    }

  }

}

[root@localhost objectstream]# cat src/main/java/com/packt/samples/JavaSerializationStream.java package com.packt.samples;

import java.io.OutputStream;

import java.io.ObjectOutputStream;

import java.io.NotSerializableException;

import java.io.IOException;

class JavaSerializationStream extends SerializationStream {

  public OutputStream out;

  public int counterReset,counter;

  public boolean extraDebugInfo;

  public ObjectOutputStream objOut;

  public JavaSerializationStream(OutputStream out,int counterReset,boolean extraDebugInfo) {

    this.out = out;

    this.counterReset = counterReset;

    this.extraDebugInfo = extraDebugInfo;

    try {

      objOut = new ObjectOutputStream(out);

    } catch(IOException e) {

      System.out.println(e);

    }

    counter = 0;

  }

  public SerializationStream writeObject(T t) {

    try {

      objOut.writeObject(t);

    } catch(NotSerializableException e) {

      System.out.println(e);

    } catch(IOException e) {

      System.out.println(e);

    }

    counter += 1;

    if (counterReset > 0 && counter >= counterReset) {

      try {

        objOut.reset();

      } catch(IOException e) {

        System.out.println(e);

      }

      counter = 0;

    }

    return this;

  }

  public void flush() {

    try {

      objOut.flush();

    } catch(IOException e) {

      System.out.println(e);

    }

  }

  public void close() {

    try {

      objOut.close();

    } catch(IOException e) {

      System.out.println(e);

    }

  }

}

8.用来保存key 和 value的 类

[root@localhost objectstream]# cat src/main/java/com/packt/samples/ObjectPair.java

package com.packt.samples;

class ObjectPair {

    private T key,value;

    public ObjectPair(T key,T value) {

        this.key = key;

        this.value = value;

    }

}

9.用于测试序列化的类

[root@localhost objectstream]# cat src/main/java/com/packt/samples/Task.java

package com.packt.samples;

import java.io.Serializable;

public class Task implements Serializable {

  public int id;

  public Task(int id) {

    this.id = id;

  }

  @Override

  public String toString() {

      return (new Integer(id)).toString();

  }

}

10.迭代器对象

[root@localhost objectstream]# cat src/main/java/com/packt/samples/NextIterator.java

package com.packt.samples;

import java.util.Iterator;

import java.util.NoSuchElementException;

abstract class NextIterator implements Iterator {

  private boolean gotNext = false;

  private U nextValue;

  private boolean closed = false;

  protected boolean finished = false;

  abstract protected U getNext();

  abstract protected void close();

  public void closeIfNeeded() {

    if (!closed) {

      closed = true;

      close();

    }

  }

  @Override

  public boolean hasNext() {

    if (!finished) {

      if (!gotNext) {

        nextValue = getNext();

        if (finished) {

          closeIfNeeded();

        }

        gotNext = true;

      }

    }

    return !finished;

  }

  @Override

  public U next() {

    if (!hasNext()) {

      throw new NoSuchElementException("End of stream");

    }

    gotNext = false;

    return nextValue;

  }

}


11.编译

mvn clean package -DskipTests

12.执行

[root@localhost objectstream]# java -jar target/objectstream-1.0.0-jar-with-dependencies.jar

1

你可能感兴趣的:(spark序列化和反序列化对象的实现,java版本)