hadoop配置相关

core-site.xml

  
    fs.defaultFS
    hdfs://earth
    true
  
  
    hadoop.tmp.dir
    /data1/tmp-security
    true
  
  
    ha.zookeeper.quorum
    hadoop-btzk0001.eniot.io:2181,hadoop-btzk0002.eniot.io:2181,hadoop-btzk0003.eniot.io:2181
  
  
    ha.failover-controller.active-standby-elector.zk.op.retries
    120
  
  
  
  
  
    
    fs.du.interval
    1200000
  
  
    fs.df.interval
    60000
  
  
  
  
  
    fs.trash.interval
    1440
    true
  
  
  
  
    io.file.buffer.size
    65536
  
  
    io.compression.codecs
    org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec
  
  
    io.compression.codec.lzo.class
    com.hadoop.compression.lzo.LzoCodec
  
  
    lzo.text.input.format.ignore.nonlzo
    false
  
  
    hadoop.ssl.hostname.verifier
    ALLOW_ALL
  

hdfs-site.xml

  
  
   
    dfs.webhdfs.enabled
    false
  
  
  
  
  
    dfs.replication
    3
  
  
    dfs.nameservices
    earth
  
  
    dfs.ha.namenodes.earth
    nn1,nn2
  
  
    dfs.ha.automatic-failover.enabled
    true
  
  
    dfs.ha.fencing.methods
    sshfence
      shell(/bin/true)
  
  
    dfs.ha.fencing.ssh.private-key-files
    /home/hdfs/.ssh/id_rsa
  
  
    dfs.ha.fencing.ssh.connect-timeout
    30000
  
  
    dfs.permissions.enabled
    true
  
  
    dfs.permissions.superusergroup
    hadoop
  
  
    dfs.cluster.administrators
    hadoop
  
  
    dfs.data.transfer.protection
    integrity
  

  
  
  
    
    dfs.blockreport.initialDelay
    180
  
  
  
  
  
  
    dfs.blockreport.intervalMsec
    21600000
  
  
  
  
    
    dfs.blockreport.split.threshold
    1000000
  
  
    
    dfs.block.scanner.volume.bytes.per.second
    1048576
  

  
  
  
    dfs.namenode.resource.du.reserved
    100G
  
  
  
    dfs.namenode.name.dir.restore
    false
  
  
    
    dfs.namenode.checkpoint.txns
    10000000
  


  
    dfs.namenode.rpc-address.earth.nn1
    hadoop-btnn0001.eniot.io:8020
      
      dfs.namenode.https-address.earth.nn1
      hadoop-btnn0001.eniot.io:50470
    
  
    dfs.namenode.rpc-address.earth.nn2
    hadoop-btnn0002.eniot.io:8020
      
      dfs.namenode.https-address.earth.nn2
      hadoop-btnn0002.eniot.io:50470
    
  
    dfs.namenode.acls.enabled
    True
  
  
    dfs.namenode.name.dir
    /data1/hdfs/hadoopNNData
    true
  
  
    dfs.namenode.shared.edits.dir
    qjournal://hadoop-btnn0001.eniot.io:8485;hadoop-btnn0002.eniot.io:8485;hadoop-btnn0003.eniot.io:8485/earth
  
  
    dfs.namenode.delegation.token.max-lifetime
    31536000000
  
  
    dfs.namenode.delegation.token.renew-interval
    31536000000
  
  
    dfs.namenode.checkpoint.period
    7200
  


  
  
    
    dfs.datanode.scan.period.hours
    504
  
  
  
    dfs.datanode.handler.count
    10
  
  
    dfs.datanode.available-space-volume-choosing-policy.balanced-space-preference-fraction
    0.75f
  
  
  
    dfs.datanode.available-space-volume-choosing-policy.balanced-space-threshold
    10737418240
  
  
    dfs.datanode.address
    0.0.0.0:50010
  
  
    dfs.datanode.http.address
    0.0.0.0:50075
  
  
    dfs.datanode.data.dir
    /data1/dfs/data
    true
  
  
   
     balancer时，hdfs移动数据的速度，默认值为1M/S的速度。一般情况下设置为10M
    50485760
  
  
  
    dfs.datanode.balance.max.concurrent.moves
    25
  
  
  
    dfs.datanode.du.reserved
    0
  

  
  
    dfs.journalnode.edits.dir
    /data1/hdfs/journal
  

  
 
  
    dfs.http.policy
    HTTPS_ONLY
  
  
    dfs.https.port
    50470
  
  

  
    dfs.image.transfer.timeout
    1800000
  
  
  
    dfs.image.transfer.bandwidthPerSec
    5242880
  
  
    dfs.hosts.exclude
    /usr/local/hadoop/hadoop-release/etc/hadoop/hosts-dfs.exclude
  
  
    dfs.block.access.token.enable
    true
  
  
    dfs.client.read.shortcircuit
    true
  
  
    dfs.domain.socket.path
    /var/lib/hadoop-hdfs/dn_socket
  
  
    dfs.client.failover.proxy.provider.earth
    org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider
  


  
    dfs.namenode.keytab.file
    /etc/security/keytab/hdfs.keytab
  
  
    dfs.namenode.kerberos.principal
    hdfs/[email protected]
  

  
    dfs.datanode.keytab.file
    /etc/security/keytab/hdfs.keytab
  
  
    dfs.datanode.kerberos.principal
    hdfs/[email protected]
  

  
    dfs.journalnode.keytab.file
    /etc/security/keytab/hdfs.keytab
  
  
    dfs.journalnode.kerberos.principal
    hdfs/[email protected]
  
  
    dfs.journalnode.kerberos.internal.spnego.principal
    HTTP/[email protected]
  

  
    dfs.web.authentication.kerberos.keytab
    /etc/security/keytab/hdfs.keytab
  
  
    dfs.web.authentication.kerberos.principal
    HTTP/[email protected]
  

  
        dfs.permissions
        true
    
    
        dfs.namenode.inode.attributes.provider.class
        org.apache.ranger.authorization.hadoop.RangerHdfsAuthorizer
    
	
	

dfs.qjournal.start-segment.timeout.ms
20000


dfs.qjournal.prepare-recovery.timeout.ms
120000


dfs.qjournal.accept-recovery.timeout.ms
120000


dfs.qjournal.finalize-segment.timeout.ms
120000


dfs.qjournal.select-input-streams.timeout.ms
20000


dfs.qjournal.get-journal-state.timeout.ms
120000


dfs.qjournal.new-epoch.timeout.ms
120000


dfs.qjournal.write-txns.timeout.ms
20000

	

yarn-site.xml

  
    yarn.http.policy
    HTTPS_ONLY
  
  
    yarn.acl.enable
    true
  
  
    yarn.admin.acl
    yarn,hadoop
  
  
    yarn.web-proxy.address
    hadoop-btnn0003.eniot.io:8089
  
  
  
    yarn.resourcemanager.zk-state-store.parent-path
    /rmstore_batch
 
  
    yarn.resourcemanager.cluster-id
    yarn-cluster-batch
  
  
    yarn.resourcemanager.recovery.enabled
    true
  
  
    yarn.resourcemanager.ha.enabled
    true
  
  
    yarn.resourcemanager.ha.rm-ids
    rm1,rm2
  
  
    yarn.resourcemanager.connect.retry-interval.ms
    2000
  
  
    yarn.resourcemanager.am.max-retries
    5
  


  
  
    yarn.resourcemanager.zk-acl
    world:anyone:rwcda
  
  
    yarn.resourcemanager.zk-address
    hadoop-btzk0001.eniot.io:2181,hadoop-btzk0002.eniot.io:2181,hadoop-btzk0003.eniot.io:2181
  
  
    yarn.resourcemanager.store.class
    org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
  

  
    yarn.resourcemanager.hostname.rm1
    hadoop-btnn0001.eniot.io
  
  
    yarn.resourcemanager.scheduler.address.rm1
    hadoop-btnn0001.eniot.io:8030
  
  
    yarn.resourcemanager.resource-tracker.address.rm1
    hadoop-btnn0001.eniot.io:8031
  
  
    yarn.resourcemanager.address.rm1
    hadoop-btnn0001.eniot.io:8032
  
  
    yarn.resourcemanager.admin.address.rm1
    hadoop-btnn0001.eniot.io:8033
  

    
    yarn.resourcemanager.webapp.https.address.rm1
    hadoop-btnn0001.eniot.io:8090
  
  
  
    yarn.resourcemanager.hostname.rm2
    hadoop-btnn0002.eniot.io
  
  
    yarn.resourcemanager.scheduler.address.rm2
    hadoop-btnn0002.eniot.io:8030
  
  
    yarn.resourcemanager.resource-tracker.address.rm2
    hadoop-btnn0002.eniot.io:8031
  
  
    yarn.resourcemanager.address.rm2
    hadoop-btnn0002.eniot.io:8032
  
  
    yarn.resourcemanager.admin.address.rm2
    hadoop-btnn0002.eniot.io:8033
  

    
    yarn.resourcemanager.webapp.https.address.rm2
    hadoop-btnn0002.eniot.io:8090
  
  
  

  
    yarn.log.server.url
    https://hadoop-btnn0003.eniot.io:19888/jobhistory/logs
  

  
    yarn.resourcemanager.nodes.exclude-path
    /usr/local/hadoop/hadoop-release/etc/hadoop/hosts-yarn.exclude
  
  
    yarn.resourcemanager.proxy-user-privileges.enabled
    true
  
  
  
  
  
    yarn.nodemanager.linux-container-executor.group
    hadoop
  
  
    yarn.nodemanager.disk-health-checker.min-healthy-disks
    0.25
  
  
    yarn.nodemanager.disk-health-checker.max-disk-utilization-per-disk-percentage
    95.0
  
  
    yarn.nodemanager.disk-health-checker.min-free-space-per-disk-mb
    0
  
  
    yarn.nodemanager.vmem-check-enabled
    false
  
  
    yarn.nodemanager.recovery.enabled
    true
  
  
    yarn.nodemanager.linux-container-executor.path
    /usr/local/hadoop/hadoop-release/bin/container-executor
  
  
    yarn.nodemanager.address
    0.0.0.0:45454
  
  
    yarn.nodemanager.local-dirs
    /data1/yarn/local
  
  
    yarn.nodemanager.log-dirs
    /data1/yarn/userlogs
  
  
    yarn.nodemanager.log.retain-seconds
    10800
  
  
    yarn.nodemanager.aux-services
    mapreduce_shuffle
  
  
    yarn.nodemanager.resource.cpu-vcores
    4
  
  
    yarn.nodemanager.resource.memory-mb
    28942
  
  
  
  
    yarn.resourcemanager.scheduler.class
    org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler
  
  
    yarn.scheduler.maximum-allocation-mb
    20480
  
  
    yarn.scheduler.minimum-allocation-mb
    128
  
  
    yarn.scheduler.increment-allocation-mb
    128
  
  
    yarn.scheduler.fair.allocation.file
    /usr/local/hadoop/hadoop-release/etc/hadoop/fair-scheduler.xml
  
  
   
    yarn.scheduler.fair.max.assign
    10
  
  
    yarn.scheduler.fair.user-as-default-queue
    false
  
  
    yarn.scheduler.fair.allow-undeclared-pools
    false
  
  
  
  
  
    yarn.log-aggregation-enable
    true
  
  
    yarn.log-aggregation.retain-seconds
    604800
  
  
    yarn.log-aggregation.retain-check-interval-seconds
    86400
  
  
    yarn.nodemanager.remote-app-log-dir
    /mr-history
  
  
    yarn.nodemanager.remote-app-log-dir-suffix
    logs
  
  
     yarn.nodemanager.localizer.cache.cleanup.interval-ms
     600000
  
  
     yarn.nodemanager.log-aggregator.on-fail.remain-log-in-sec
     259200
  


  
    yarn.nodemanager.principal
    yarn/[email protected]
  
  
    yarn.nodemanager.keytab
    /etc/security/keytab/yarn.keytab
  
  
    yarn.resourcemanager.keytab
    /etc/security/keytab/yarn.keytab
  
  
    yarn.resourcemanager.principal
    yarn/[email protected]
  
  
    yarn.web-proxy.keytab
    /etc/security/keytab/yarn.keytab
  
  
    yarn.web-proxy.principal
    HTTP/[email protected]
  


  yarn.nodemanager.container-executor.class
  org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor

mapred-site.xm

  
    yarn.app.mapreduce.am.resource.mb
    2048
  
  
    mapreduce.framework.name
    yarn
  
  
    mapreduce.cluster.local.dir
    /data1/yarn/mapred/data
  
  
    mapreduce.task.io.sort.mb
    300
  
  
    mapred.child.env
    LD_LIBRARY_PATH=/usr/local/hadoop/hadoop-release/lib/libexec
  
  
    mapred.child.java.opts
    -Xmx3072m -XX:-UseGCOverheadLimit
  
  
    mapreduce.tasktracker.http.address
    0.0.0.0:50060
  
  
  
    
    mapreduce.task.userlog.limit.kb
    51200
  
  
  
    mapreduce.reduce.shuffle.memory.limit.percent
    0.25
  
  
    yarn.app.mapreduce.task.container.log.backups
    1
  
  
    yarn.app.mapreduce.am.staging-dir
    /tmp/hadoop-yarn/staging
  
  
    mapreduce.jobhistory.done-dir
    ${yarn.app.mapreduce.am.staging-dir}/history/done
  
  
    mapreduce.jobhistory.address
    hadoop-btnn0003.eniot.io:10020
  
  
    mapreduce.jobhistory.cleaner.enable
    true
  
  
    mapreduce.jobhistory.max-age-ms
    604800000
  
  
    mapreduce.jobhistory.cleaner.interval-ms
    86400000
  
  
    mapreduce.jobhistory.move.interval-ms
    180000
  
  
    mapreduce.jobhistory.http.policy
    HTTPS_ONLY
  
  
    mapreduce.jobhistory.webapp.https.address
    hadoop-btnn0003.eniot.io:19888
  
  
  
  
    mapreduce.map.speculative
    false
  
  
    mapreduce.reduce.speculative
    false
  
  
    mapreduce.map.output.compress
    true
  
  
    mapreduce.map.output.compress.codec
    com.hadoop.compression.lzo.LzoCodec
  
  
  
    mapreduce.map.memory.mb
    2048
  
  
    mapreduce.reduce.memory.mb
    4096
  
  
    mapreduce.map.java.opts
    -Duser.home=. -Djava.net.preferIPv4Stack=true -Xmx1800m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=15 -XX:CMSInitiatingOccupancyFraction=70
  
  
    mapreduce.reduce.java.opts
    -Duser.home=. -Djava.net.preferIPv4Stack=true -Xmx3600m -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+UseCMSCompactAtFullCollection -XX:CMSFullGCsBeforeCompaction=15 -XX:CMSInitiatingOccupancyFraction=70
  


  
    mapreduce.jobhistory.keytab
    /etc/security/keytab/mapred.keytab
  
  
    mapreduce.jobhistory.principal
    mapred/[email protected]
  

fail-schduler.xml

0 mb, 0 vcores
0 mb, 0 vcores
50
fifo
1.0
600


32768 mb, 5 vcores
163840 mb, 24 vcores
50
fifo
1.0
600


0 mb, 0 vcores
0 mb, 0 vcores
50
fifo
1.0
600


32768 mb, 16 vcores
32768 mb, 16 vcores
50
fifo
1.0
600


0 mb, 0 vcores
8192 mb, 4 vcores
10
fifo
1.0
600

other

• dfs.datanode.max.transfer.threads: 数据传输最大线程，默认：16384

• dfs.datanode.ipc.address：
  DN的IPC监听端口，写0的话监听在随机端口通过心跳传输给NN，默认：8010

• dfs.namenode.checkpoint.dir：
  standy NN节点存储 checkpoint
  文件目录，默认：/hadoop/hdfs/namesecondary

• dfs.namenode.stale.datanode.interval:
  标记一个dn为“down机”时间间隔,即:如果
  nn没有接到一个dn心跳超过这个时间间隔,dn将标记为“过期”,过期的间隔不能太小 (默认: 3000)

• dfs.namenode.safemode.threshold-pct
  指定应有多少比例的数据块满足最小副本数要求,默认0.999f 说明：
  (NN在启动的时候首先进入安全模式，如果DN丢失的block达到此参数设置的比例，则系统会一直处于安全模式状态即只读状态。如果设为1则HDFS永远是处于SafeMode）