Protobuf知识积累

背景

有时候逆向会遇到protobuf相关的东西，这篇文章主要是记录一下分析过程

Protobuf基本使用

1.定义proto文件

//定义版本
syntax = "proto2";

//java相关定义
package netty;
option java_package = "com.test";
option java_outer_classname = "TestProto";

//Protobuf定义
message SubscribeReq{
	//optional/required/repeated fieldType fieldName = 标志符
    optional bool userName = 1;
    optional double productName = 2;
    optional float address1 = 3;
    optional int32 address2 = 4;
    optional int64 address3 = 5;
    optional uint32 address4 = 6;
    optional uint64 address5 = 7;
    optional sint32 address6 = 8;
    optional sint64 address7 = 9;
    optional fixed64 address8 = 10;
    required fixed32 address9 = 11;
    required sfixed32 address11 = 12;
    required sfixed64 address112 = 13;
    required string address12 = 14;
    required bytes address1223 = 15;
    message test{

    }
    required test test_1 = 16;
    enum PhoneType {
         MOBILE = 0;
         HOME = 1;
         WORK = 2;
     }
     required PhoneType pPhoneType = 17;
}

2. 编译命令

protoc --proto_path=. --java_out=lite:. java.proto
protoc -I=./ --python_out=./ ./ocr.proto

4. 测试写入

from com.test import SubscribeReq

def main():
    hw = SubscribeReq()
    hw.id = 124
    hw.str = "bbb"
    hw.strr = "aaa"

    hw.hah_d.pp = 110
    hw.hah_d.cc = 0.1
    hw.hah_d.bb = True

    with open("test.pb", "wb") as f:
        f.write(hw.SerializeToString())

if __name__ == "__main__":
    main()% 

6. 测试读取

import ocr_pb2
import sys
def dealIt(ocr):
    print("Ocr result: " + ocr.ocrbody.ocrresult.result)

ocr = ocr_pb2.ocr()

f = open(sys.argv[1],"rb")
ocr.ParseFromString(f.read())
f.close()

dealIt(ocr)

还原proto文件

1. 获取二进制流protobuf文件
2. 解码命令：
   cat data.pb | protoc --decode_raw
3. 获取输出类似

1: 124
2: "bbb"
4: "aaa"
5 {
  1: 110
  2: 0x3dcccccd
  3: 1
}

4. 还原proto文件

根据输出，直接还原，前面数字为标识号，后面为数值，根据数值简单推断字段类型，尽量使用optionnal

准确逆向还原proto文件中字段类型

protobuf lite

1. 找到类似如下的字符串:
   \u0001\u0002\u0000\u0001\u0001\u0002\u0002\u0000\u0000\u0000\u0001\u1008\u0000\u0002\u1008\u0001

2. 使用以下Java类

package cn.tasfa.ppcodetest;

import android.util.Log;
import java.util.HashMap;

public class ProtoBufLiteType {

    static HashMap<Integer,String> fieldTypeMap;

    static {
        fieldTypeMap = new HashMap<>();
        fieldTypeMap.put(0,"double");
        fieldTypeMap.put(1,"float");
        fieldTypeMap.put(2,"int64");
        fieldTypeMap.put(3,"uint64");
        fieldTypeMap.put(4,"int32");
        fieldTypeMap.put(5,"fixed64");
        fieldTypeMap.put(6,"fixed32");
        fieldTypeMap.put(7,"bool");
        fieldTypeMap.put(8,"string");
        fieldTypeMap.put(9,"message");
        fieldTypeMap.put(10,"bytes");
        fieldTypeMap.put(11,"uint32");
        fieldTypeMap.put(12,"enum");
        fieldTypeMap.put(13,"sfixed32");
        fieldTypeMap.put(14,"sfixed64");
        fieldTypeMap.put(15,"sint32");
        fieldTypeMap.put(16,"sint64");
    }

    private static final String TAG = "ProtoBufLiteType";
    static final int ONEOF_TYPE_OFFSET = 51 /* FieldType.MAP + 1 */;

    public static void deal(){
        dealV2033();
        //dealV2021();
    }

    public static void getFiledType(String info){
        final int length = info.length();
            int i = 0;

            int next = info.charAt(i++);
            if (next >= 0xD800) {
                int result = next & 0x1FFF;
                int shift = 13;
                while ((next = info.charAt(i++)) >= 0xD800) {
                    result |= (next & 0x1FFF) << shift;
                    shift += 13;
                }
                next = result | (next << shift);
            }
            final int flags = next;

            next = info.charAt(i++);
            if (next >= 0xD800) {
                int result = next & 0x1FFF;
                int shift = 13;
                while ((next = info.charAt(i++)) >= 0xD800) {
                    result |= (next & 0x1FFF) << shift;
                    shift += 13;
                }
                next = result | (next << shift);
            }
            final int fieldCount = next;
        Log.d(TAG, "fieldCount: " + fieldCount);

            final int oneofCount;
            final int hasBitsCount;
            final int minFieldNumber;
            final int maxFieldNumber;
            final int numEntries;
            final int mapFieldCount;
            final int repeatedFieldCount;
            final int checkInitialized;
            final int[] intArray;
            int objectsPosition;
            if (fieldCount == 0) {
                oneofCount = 0;
                hasBitsCount = 0;
                minFieldNumber = 0;
                maxFieldNumber = 0;
                numEntries = 0;
                mapFieldCount = 0;
                repeatedFieldCount = 0;
                checkInitialized = 0;
                intArray = null;
                objectsPosition = 0;
            } else {
                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                oneofCount = next;
                //Log.d(TAG, "oneofCount: " + oneofCount);

                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                hasBitsCount = next;


                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                minFieldNumber = next;

                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                maxFieldNumber = next;

                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                numEntries = next;

                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                mapFieldCount = next;

                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                repeatedFieldCount = next;
                //Log.d(TAG, "repeatedFieldCount: " + repeatedFieldCount);


                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                checkInitialized = next;
                intArray = new int[checkInitialized + mapFieldCount + repeatedFieldCount];
                // Field objects are after a list of (oneof, oneofCase) pairs  + a list of hasbits fields.
                objectsPosition = oneofCount * 2 + hasBitsCount;
            }

//            final sun.misc.Unsafe unsafe = UNSAFE;
//            final Object[] messageInfoObjects = messageInfo.getObjects();
//            int checkInitializedPosition = 0;
//            final Class messageClass = messageInfo.getDefaultInstance().getClass();
//            int[] buffer = new int[numEntries * INTS_PER_FIELD];
//            Object[] objects = new Object[numEntries * 2];

            int mapFieldIndex = checkInitialized;
            int repeatedFieldIndex = checkInitialized + mapFieldCount;

            int bufferIndex = 0;
            while (i < length) {
                final int fieldNumber;
                final int fieldTypeWithExtraBits;
                final int fieldType;

                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                fieldNumber = next;
                Log.e(TAG, "fieldNumber: " + fieldNumber);


                next = info.charAt(i++);
                if (next >= 0xD800) {
                    int result = next & 0x1FFF;
                    int shift = 13;
                    while ((next = info.charAt(i++)) >= 0xD800) {
                        result |= (next & 0x1FFF) << shift;
                        shift += 13;
                    }
                    next = result | (next << shift);
                }
                fieldTypeWithExtraBits = next;
                fieldType = fieldTypeWithExtraBits & 0xFF;
                //Log.e(TAG, "fieldType num: " + fieldType);
                Log.e(TAG, "fieldType: " + fieldTypeMap.get(fieldType));


                final int fieldOffset;
                final int presenceMaskShift;
                final int presenceFieldOffset;

                // Oneof
                if (fieldType >= ONEOF_TYPE_OFFSET) {
                    next = info.charAt(i++);
                    if (next >= 0xD800) {
                        int result = next & 0x1FFF;
                        int shift = 13;
                        while ((next = info.charAt(i++)) >= 0xD800) {
                            result |= (next & 0x1FFF) << shift;
                            shift += 13;
                        }
                        next = result | (next << shift);
                    }
                    int oneofIndex = next;

                    final int oneofFieldType = fieldType - ONEOF_TYPE_OFFSET;
                    if (oneofFieldType == 9 /* FieldType.MESSAGE */
                            || oneofFieldType == 17 /* FieldType.GROUP */) {
                        //objects[bufferIndex / INTS_PER_FIELD * 2 + 1] = messageInfoObjects[objectsPosition++];
                    } else if (oneofFieldType == 12 /* FieldType.ENUM */) {
                        // proto2
                        if ((flags & 0x1) == 0x1) {
                            //objects[bufferIndex / INTS_PER_FIELD * 2 + 1] = messageInfoObjects[objectsPosition++];
                        }
                    }

                    final java.lang.reflect.Field oneofField;
                    int index = oneofIndex * 2;
//                    Object o = messageInfoObjects[index];
//                    if (o instanceof java.lang.reflect.Field) {
//                        oneofField = (java.lang.reflect.Field) o;
//                    } else {
//                        oneofField = reflectField(messageClass, (String) o);
//                        // Memoize java.lang.reflect.Field instances for oneof/hasbits fields, since they're
//                        // potentially used for many Protobuf fields.  Since there's a 1-1 mapping from the
//                        // Protobuf field to the Java Field for non-oneofs, there's no benefit for memoizing
//                        // those.
//                        messageInfoObjects[index] = oneofField;
//                    }

                    //fieldOffset = (int) unsafe.objectFieldOffset(oneofField);

                    final java.lang.reflect.Field oneofCaseField;
                    index++;
//                    o = messageInfoObjects[index];
//                    if (o instanceof java.lang.reflect.Field) {
//                        oneofCaseField = (java.lang.reflect.Field) o;
//                    } else {
//                        oneofCaseField = reflectField(messageClass, (String) o);
//                        messageInfoObjects[index] = oneofCaseField;
//                    }

                    //presenceFieldOffset = (int) unsafe.objectFieldOffset(oneofCaseField);
                    //presenceMaskShift = 0;
                } else {
//                    Field field = reflectField(messageClass, (String) messageInfoObjects[objectsPosition++]);
//                    if (fieldType == 9 /* FieldType.MESSAGE */ || fieldType == 17 /* FieldType.GROUP */) {
//                        objects[bufferIndex / INTS_PER_FIELD * 2 + 1] = field.getType();
//                    } else if (fieldType == 27 /* FieldType.MESSAGE_LIST */
//                            || fieldType == 49 /* FieldType.GROUP_LIST */) {
//                        objects[bufferIndex / INTS_PER_FIELD * 2 + 1] = messageInfoObjects[objectsPosition++];
//                    } else if (fieldType == 12 /* FieldType.ENUM */
//                            || fieldType == 30 /* FieldType.ENUM_LIST */
//                            || fieldType == 44 /* FieldType.ENUM_LIST_PACKED */) {
//                        if ((flags & 0x1) == 0x1) {
//                            objects[bufferIndex / INTS_PER_FIELD * 2 + 1] = messageInfoObjects[objectsPosition++];
//                        }
//                    } else if (fieldType == 50 /* FieldType.MAP */) {
//                        intArray[mapFieldIndex++] = bufferIndex;
//                        objects[bufferIndex / INTS_PER_FIELD * 2] = messageInfoObjects[objectsPosition++];
//                        if ((fieldTypeWithExtraBits & 0x800) != 0) {
//                            objects[bufferIndex / INTS_PER_FIELD * 2 + 1] = messageInfoObjects[objectsPosition++];
//                        }
//                    }

                    //fieldOffset = (int) unsafe.objectFieldOffset(field);
                    if ((flags & 0x1) == 0x1 && fieldType <= 17 /* FieldType.GROUP */) {
                        next = info.charAt(i++);
                        if (next >= 0xD800) {
                            int result = next & 0x1FFF;
                            int shift = 13;
                            while ((next = info.charAt(i++)) >= 0xD800) {
                                result |= (next & 0x1FFF) << shift;
                                shift += 13;
                            }
                            next = result | (next << shift);
                        }
                        int hasBitsIndex = next;

                        final java.lang.reflect.Field hasBitsField;
                        int index = oneofCount * 2 + hasBitsIndex / 32;
                        //Object o = messageInfoObjects[index];
//                        if (o instanceof java.lang.reflect.Field) {
//                            hasBitsField = (java.lang.reflect.Field) o;
//                        } else {
//                            //hasBitsField = reflectField(messageClass, (String) o);
//                            //messageInfoObjects[index] = hasBitsField;
//                        }

                        //presenceFieldOffset = (int) unsafe.objectFieldOffset(hasBitsField);
                        presenceMaskShift = hasBitsIndex % 32;
                    } else {
                        presenceFieldOffset = 0;
                        presenceMaskShift = 0;
                    }

                    if (fieldType >= 18 && fieldType <= 49) {
                        // Field types of repeated fields are in a consecutive range from 18 (DOUBLE_LIST) to
                        // 49 (GROUP_LIST).
                        //intArray[repeatedFieldIndex++] = fieldOffset;
                    }
                }
            }
    }
}

逆向获取protobuf版本

关键点

1. "This program was compiled against version "
2. #define PROTOBUF_VERSION 3008000
3. port_def.inc

protobuf编译

PC编译

• ./autogen.sh
• ./configure --prefix=/usr/test
• make -j12
• make install

Android编译

1. $NDK_ROOT/build/tools/make-standalone-toolchain.sh --arch=arm --platform=android-21 --toolchain=arm-linux-android-clang5.0 --install-dir=$toolchain_dir/arm-21-toolchain-clang-32 --use-llvm --stl=libc++

git clone https://github.com/protocolbuffers/protobuf.git
git checkout v3.10.1
git submodule update --init --recursive

3. ./autogen.sh

export build_dir=`pwd`/../libprotobuf/android
export sysroot=`pwd`/../arm-21-toolchain-clang-32/sysroot   ##前面ndk生成的clang-32路径
export PATH=`pwd`/../arm-21-toolchain-clang-32/bin:$PATH
export CC="arm-linux-androideabi-clang --sysroot $sysroot"
export CXX="arm-linux-androideabi-clang++ --sysroot $sysroot"

./configure \
--host=arm-linux-androideabi \
--with-protoc=protoc \
--with-sysroot="$sysroot" \
--disable-shared \
--prefix="$build_dir/armeabi-v7a" \
--enable-cross-compile \
CFLAGS="-march=armv7-a -D__ANDROID_API__=21" \
CXXFLAGS="-frtti -fexceptions -march=armv7-a -D__ANDROID_API__=21" \
LIBS="-llog -lz -lc++_static"

make -j12
make install

protobuf

同理

参考

MessageSchema.java

Protobuf之proto文件编写规则

c-Protobuf ：编写proto文件以及protobuf文件的使用

Protobuf 3 定义复杂的proto文件

protobuf流的反解析Message

Protocol Buffers C++ 入门教程

Android之protobuf lib库