HDFS Inner Protocol
hdfs.proto
/**
* Extended block idenfies a block
*/
message ExtendedBlockProto {
required string poolId = 1; // Block pool id - gloablly unique across clusters
required uint64 blockId = 2; // the local id within a pool
required uint64 generationStamp = 3;
optional uint64 numBytes = 4 [default = 0]; // len does not belong in ebid
// here for historical reasons
}
/**
* Identifies a Datanode
*/
message DatanodeIDProto {
required string ipAddr = 1; // IP address
required string hostName = 2; // hostname
required string datanodeUuid = 3; // UUID assigned to the Datanode. For
// upgraded clusters this is the same
// as the original StorageID of the
// Datanode.
required uint32 xferPort = 4; // data streaming port
required uint32 infoPort = 5; // datanode http port
required uint32 ipcPort = 6; // ipc server port
optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
}
/**
* Datanode local information
*/
message DatanodeLocalInfoProto {
required string softwareVersion = 1;
required string configVersion = 2;
required uint64 uptime = 3;
}
/**
* DatanodeInfo array
*/
message DatanodeInfosProto {
repeated DatanodeInfoProto datanodes = 1;
}
/**
* The status of a Datanode
*/
message DatanodeInfoProto {
required DatanodeIDProto id = 1;
optional uint64 capacity = 2 [default = 0];
optional uint64 dfsUsed = 3 [default = 0];
optional uint64 remaining = 4 [default = 0];
optional uint64 blockPoolUsed = 5 [default = 0];
optional uint64 lastUpdate = 6 [default = 0];
optional uint32 xceiverCount = 7 [default = 0];
optional string location = 8;
enum AdminState {
NORMAL = 0;
DECOMMISSION_INPROGRESS = 1;
DECOMMISSIONED = 2;
}
optional AdminState adminState = 10 [default = NORMAL];
optional uint64 cacheCapacity = 11 [default = 0];
optional uint64 cacheUsed = 12 [default = 0];
}
/**
* Summary of a file or directory
*/
message ContentSummaryProto {
required uint64 length = 1;
required uint64 fileCount = 2;
required uint64 directoryCount = 3;
required uint64 quota = 4;
required uint64 spaceConsumed = 5;
required uint64 spaceQuota = 6;
}
/**
* Contains a list of paths corresponding to corrupt files and a cookie
* used for iterative calls to NameNode.listCorruptFileBlocks.
*
*/
message CorruptFileBlocksProto {
repeated string files = 1;
required string cookie = 2;
}
/**
* File or Directory permision - same spec as posix
*/
message FsPermissionProto {
required uint32 perm = 1; // Actually a short - only 16bits used
}
/**
* Types of recognized storage media.
*/
enum StorageTypeProto {
DISK = 1;
SSD = 2;
}
/**
* A list of storage IDs.
*/
message StorageUuidsProto {
repeated string storageUuids = 1;
}
/**
* A LocatedBlock gives information about a block and its location.
*/
message LocatedBlockProto {
required ExtendedBlockProto b = 1;
required uint64 offset = 2; // offset of first byte of block in the file
repeated DatanodeInfoProto locs = 3; // Locations ordered by proximity to client ip
required bool corrupt = 4; // true if all replicas of a block are corrupt, else false
// If block has few corrupt replicas, they are filtered and
// their locations are not part of this object
required hadoop.common.TokenProto blockToken = 5;
repeated bool isCached = 6 [packed=true]; // if a location in locs is cached
repeated StorageTypeProto storageTypes = 7;
repeated string storageIDs = 8;
}
message DataEncryptionKeyProto {
required uint32 keyId = 1;
required string blockPoolId = 2;
required bytes nonce = 3;
required bytes encryptionKey = 4;
required uint64 expiryDate = 5;
optional string encryptionAlgorithm = 6;
}
/**
* A set of file blocks and their locations.
*/
message LocatedBlocksProto {
required uint64 fileLength = 1;
repeated LocatedBlockProto blocks = 2;
required bool underConstruction = 3;
optional LocatedBlockProto lastBlock = 4;
required bool isLastBlockComplete = 5;
}
/**
* Status of a file, directory or symlink
* Optionally includes a file's block locations if requested by client on the rpc call.
*/
message HdfsFileStatusProto {
enum FileType {
IS_DIR = 1;
IS_FILE = 2;
IS_SYMLINK = 3;
}
required FileType fileType = 1;
required bytes path = 2; // local name of inode encoded java UTF8
required uint64 length = 3;
required FsPermissionProto permission = 4;
required string owner = 5;
required string group = 6;
required uint64 modification_time = 7;
required uint64 access_time = 8;
// Optional fields for symlink
optional bytes symlink = 9; // if symlink, target encoded java UTF8
// Optional fields for file
optional uint32 block_replication = 10 [default = 0]; // only 16bits used
optional uint64 blocksize = 11 [default = 0];
optional LocatedBlocksProto locations = 12; // suppled only if asked by client
// Optional field for fileId
optional uint64 fileId = 13 [default = 0]; // default as an invalid id
optional int32 childrenNum = 14 [default = -1];
}
/**
* Checksum algorithms/types used in HDFS
* Make sure this enum's integer values match enum values' id properties defined
* in org.apache.hadoop.util.DataChecksum.Type
*/
enum ChecksumTypeProto {
CHECKSUM_NULL = 0;
CHECKSUM_CRC32 = 1;
CHECKSUM_CRC32C = 2;
}
/**
* HDFS Server Defaults
*/
message FsServerDefaultsProto {
required uint64 blockSize = 1;
required uint32 bytesPerChecksum = 2;
required uint32 writePacketSize = 3;
required uint32 replication = 4; // Actually a short - only 16 bits used
required uint32 fileBufferSize = 5;
optional bool encryptDataTransfer = 6 [default = false];
optional uint64 trashInterval = 7 [default = 0];
optional ChecksumTypeProto checksumType = 8 [default = CHECKSUM_CRC32];
}
/**
* Directory listing
*/
message DirectoryListingProto {
repeated HdfsFileStatusProto partialListing = 1;
required uint32 remainingEntries = 2;
}
/**
* Status of a snapshottable directory: besides the normal information for
* a directory status, also include snapshot quota, number of snapshots, and
* the full path of the parent directory.
*/
message SnapshottableDirectoryStatusProto {
required HdfsFileStatusProto dirStatus = 1;
// Fields specific for snapshottable directory
required uint32 snapshot_quota = 2;
required uint32 snapshot_number = 3;
required bytes parent_fullpath = 4;
}
/**
* Snapshottable directory listing
*/
message SnapshottableDirectoryListingProto {
repeated SnapshottableDirectoryStatusProto snapshottableDirListing = 1;
}
/**
* Snapshot diff report entry
*/
message SnapshotDiffReportEntryProto {
required bytes fullpath = 1;
required string modificationLabel = 2;
optional bytes targetPath = 3;
}
/**
* Snapshot diff report
*/
message SnapshotDiffReportProto {
// full path of the directory where snapshots were taken
required string snapshotRoot = 1;
required string fromSnapshot = 2;
required string toSnapshot = 3;
repeated SnapshotDiffReportEntryProto diffReportEntries = 4;
}
/**
* Common node information shared by all the nodes in the cluster
*/
message StorageInfoProto {
required uint32 layoutVersion = 1; // Layout version of the file system
required uint32 namespceID = 2; // File system namespace ID
required string clusterID = 3; // ID of the cluster
required uint64 cTime = 4; // File system creation time
}
/**
* Information sent by a namenode to identify itself to the primary namenode.
*/
message NamenodeRegistrationProto {
required string rpcAddress = 1; // host:port of the namenode RPC address
required string httpAddress = 2; // host:port of the namenode http server
enum NamenodeRoleProto {
NAMENODE = 1;
BACKUP = 2;
CHECKPOINT = 3;
}
required StorageInfoProto storageInfo = 3; // Node information
optional NamenodeRoleProto role = 4 [default = NAMENODE]; // Namenode role
}
/**
* Unique signature to identify checkpoint transactions.
*/
message CheckpointSignatureProto {
required string blockPoolId = 1;
required uint64 mostRecentCheckpointTxId = 2;
required uint64 curSegmentTxId = 3;
required StorageInfoProto storageInfo = 4;
}
/**
* Command sent from one namenode to another namenode.
*/
message NamenodeCommandProto {
enum Type {
NamenodeCommand = 0; // Base command
CheckPointCommand = 1; // Check point command
}
required uint32 action = 1;
required Type type = 2;
optional CheckpointCommandProto checkpointCmd = 3;
}
/**
* Command returned from primary to checkpointing namenode.
* This command has checkpoint signature that identifies
* checkpoint transaction and is needed for further
* communication related to checkpointing.
*/
message CheckpointCommandProto {
// Unique signature to identify checkpoint transation
required CheckpointSignatureProto signature = 1;
// If true, return transfer image to primary upon the completion of checkpoint
required bool needToReturnImage = 2;
}
/**
* Block information
*
* Please be wary of adding additional fields here, since INodeFiles
* need to fit in PB's default max message size of 64MB.
* We restrict the max # of blocks per file
* (dfs.namenode.fs-limits.max-blocks-per-file), but it's better
* to avoid changing this.
*/
message BlockProto {
required uint64 blockId = 1;
required uint64 genStamp = 2;
optional uint64 numBytes = 3 [default = 0];
}
/**
* Block and datanodes where is it located
*/
message BlockWithLocationsProto {
required BlockProto block = 1; // Block
repeated string datanodeUuids = 2; // Datanodes with replicas of the block
repeated string storageUuids = 3; // Storages with replicas of the block
}
/**
* List of block with locations
*/
message BlocksWithLocationsProto {
repeated BlockWithLocationsProto blocks = 1;
}
/**
* Editlog information with available transactions
*/
message RemoteEditLogProto {
required uint64 startTxId = 1; // Starting available edit log transaction
required uint64 endTxId = 2; // Ending available edit log transaction
optional bool isInProgress = 3 [default = false];
}
/**
* Enumeration of editlogs available on a remote namenode
*/
message RemoteEditLogManifestProto {
repeated RemoteEditLogProto logs = 1;
}
/**
* Namespace information that describes namespace on a namenode
*/
message NamespaceInfoProto {
required string buildVersion = 1; // Software revision version (e.g. an svn or git revision)
required uint32 unused = 2; // Retained for backward compatibility
required string blockPoolID = 3; // block pool used by the namespace
required StorageInfoProto storageInfo = 4;// Node information
required string softwareVersion = 5; // Software version number (e.g. 2.0.0)
}
/**
* Block access token information
*/
message BlockKeyProto {
required uint32 keyId = 1; // Key identifier
required uint64 expiryDate = 2; // Expiry time in milliseconds
optional bytes keyBytes = 3; // Key secret
}
/**
* Current key and set of block keys at the namenode.
*/
message ExportedBlockKeysProto {
required bool isBlockTokenEnabled = 1;
required uint64 keyUpdateInterval = 2;
required uint64 tokenLifeTime = 3;
required BlockKeyProto currentKey = 4;
repeated BlockKeyProto allKeys = 5;
}
/**
* State of a block replica at a datanode
*/
enum ReplicaStateProto {
FINALIZED = 0; // State of a replica when it is not modified
RBW = 1; // State of replica that is being written to
RWR = 2; // State of replica that is waiting to be recovered
RUR = 3; // State of replica that is under recovery
TEMPORARY = 4; // State of replica that is created for replication
}
/**
* Block that needs to be recovered with at a given location
*/
message RecoveringBlockProto {
required uint64 newGenStamp = 1; // New genstamp post recovery
required LocatedBlockProto block = 2; // Block to be recovered
}
/**
* void request
*/
message VersionRequestProto {
}
/**
* Version response from namenode.
*/
message VersionResponseProto {
required NamespaceInfoProto info = 1;
}
/**
* Information related to a snapshot
* TODO: add more information
*/
message SnapshotInfoProto {
required string snapshotName = 1;
required string snapshotRoot = 2;
required FsPermissionProto permission = 3;
required string owner = 4;
required string group = 5;
required string createTime = 6;
// TODO: do we need access time?
}
/**
* Rolling upgrade status
*/
message RollingUpgradeStatusProto {
required string blockPoolId = 1;
}
NamenodeProtocol.proto
/**
* Get list of blocks for a given datanode with the total length
* of adding up to given size
* datanode - Datanode ID to get list of block from
* size - size to which the block lengths must add up to
*/
message GetBlocksRequestProto {
required DatanodeIDProto datanode = 1; // Datanode ID
required uint64 size = 2; // Size in bytes
}
/**
* blocks - List of returned blocks
*/
message GetBlocksResponseProto {
required BlocksWithLocationsProto blocks = 1; // List of blocks
}
/**
* void request
*/
message GetBlockKeysRequestProto {
}
/**
* keys - Information about block keys at the active namenode
*/
message GetBlockKeysResponseProto {
optional ExportedBlockKeysProto keys = 1;
}
/**
* void request
*/
message GetTransactionIdRequestProto {
}
/**
* txId - Transaction ID of the most recently persisted edit log record
*/
message GetTransactionIdResponseProto {
required uint64 txId = 1; // Transaction ID
}
/**
* void request
*/
message RollEditLogRequestProto {
}
/**
* signature - A unique token to identify checkpoint transaction
*/
message RollEditLogResponseProto {
required CheckpointSignatureProto signature = 1;
}
/**
* void request
*/
message GetMostRecentCheckpointTxIdRequestProto {
}
message GetMostRecentCheckpointTxIdResponseProto{
required uint64 txId = 1;
}
/**
* registration - Namenode reporting the error
* errorCode - error code indicating the error
* msg - Free text description of the error
*/
message ErrorReportRequestProto {
required NamenodeRegistrationProto registration = 1; // Registration info
required uint32 errorCode = 2; // Error code
required string msg = 3; // Error message
}
/**
* void response
*/
message ErrorReportResponseProto {
}
/**
* registration - Information of the namenode registering with primary namenode
*/
message RegisterRequestProto {
required NamenodeRegistrationProto registration = 1; // Registration info
}
/**
* registration - Updated registration information of the newly registered
* datanode.
*/
message RegisterResponseProto {
required NamenodeRegistrationProto registration = 1; // Registration info
}
/**
* Start checkpoint request
* registration - Namenode that is starting the checkpoint
*/
message StartCheckpointRequestProto {
required NamenodeRegistrationProto registration = 1; // Registration info
}
/**
* command - Command returned by the active namenode to be
* be handled by the caller.
*/
message StartCheckpointResponseProto {
required NamenodeCommandProto command = 1;
}
/**
* End or finalize the previously started checkpoint
* registration - Namenode that is ending the checkpoint
* signature - unique token to identify checkpoint transaction,
* that was received when checkpoint was started.
*/
message EndCheckpointRequestProto {
required NamenodeRegistrationProto registration = 1; // Registration info
required CheckpointSignatureProto signature = 2;
}
/**
* void response
*/
message EndCheckpointResponseProto {
}
/**
* sinceTxId - return the editlog information for transactions >= sinceTxId
*/
message GetEditLogManifestRequestProto {
required uint64 sinceTxId = 1; // Transaction ID
}
/**
* manifest - Enumeration of editlogs from namenode for
* logs >= sinceTxId in the request
*/
message GetEditLogManifestResponseProto {
required RemoteEditLogManifestProto manifest = 1;
}
/**
* Protocol used by the sub-ordinate namenode to send requests
* the active/primary namenode.
*
* See the request and response for details of rpc call.
*/
service NamenodeProtocolService {
/**
* Get list of blocks for a given datanode with length
* of blocks adding up to given size.
*/
rpc getBlocks(GetBlocksRequestProto) returns(GetBlocksResponseProto);
/**
* Get the current block keys
*/
rpc getBlockKeys(GetBlockKeysRequestProto) returns(GetBlockKeysResponseProto);
/**
* Get the transaction ID of the most recently persisted editlog record
*/
rpc getTransactionId(GetTransactionIdRequestProto)
returns(GetTransactionIdResponseProto);
/**
* Get the transaction ID of the most recently persisted editlog record
*/
rpc getMostRecentCheckpointTxId(GetMostRecentCheckpointTxIdRequestProto)
returns(GetMostRecentCheckpointTxIdResponseProto);
/**
* Close the current editlog and open a new one for checkpointing purposes
*/
rpc rollEditLog(RollEditLogRequestProto) returns(RollEditLogResponseProto);
/**
* Request info about the version running on this NameNode
*/
rpc versionRequest(VersionRequestProto) returns(VersionResponseProto);
/**
* Report from a sub-ordinate namenode of an error to the active namenode.
* Active namenode may decide to unregister the reporting namenode
* depending on the error.
*/
rpc errorReport(ErrorReportRequestProto) returns(ErrorReportResponseProto);
/**
* Request to register a sub-ordinate namenode
*/
rpc registerSubordinateNamenode(RegisterRequestProto) returns(RegisterResponseProto);
/**
* Request to start a checkpoint.
*/
rpc startCheckpoint(StartCheckpointRequestProto)
returns(StartCheckpointResponseProto);
/**
* End of finalize the previously started checkpoint
*/
rpc endCheckpoint(EndCheckpointRequestProto)
returns(EndCheckpointResponseProto);
/**
* Get editlog manifests from the active namenode for all the editlogs
*/
rpc getEditLogManifest(GetEditLogManifestRequestProto)
returns(GetEditLogManifestResponseProto);
}