hadoop2.2.0 边角料（一）

hadoop dfs -copyFromLocal 是如何执行的？

1. bin/hadoop脚本调用hdfs脚本
2. hdfs调用org.apache.hadoop.fs.FsShell类
3. FssShell的main函数调用

   res = ToolRunner.run(shell, argv);//shell是FsShell的一个实例

4. ToolRunner.run调用的是FsShell中的run函数，其中

   instance = commandFactory.getInstance(cmd);
   if (instance == null) {
     throw new UnknownCommandException();
   }
   exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));

   copyCommands.java在CommandFactory中注册copy相关的命令

    public static void registerCommands(CommandFactory factory) {
       factory.addClass(Merge.class, "-getmerge");
       factory.addClass(Cp.class, "-cp");
       factory.addClass(CopyFromLocal.class, "-copyFromLocal");
       factory.addClass(CopyToLocal.class, "-copyToLocal");
       factory.addClass(Get.class, "-get");
       factory.addClass(Put.class, "-put");
       factory.addClass(AppendToFile.class, "-appendToFile");
     }

   
   
5. copyFromLocal类<=Put类<=CommandWithDestination类<=FsCommand类

   @Override
       protected void processOptions(LinkedList args) throws IOException {
         CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE, "f", "p");
         cf.parse(args);
         setOverwrite(cf.getOpt("f"));
         setPreserve(cf.getOpt("p"));
         getRemoteDestination(args);
         // should have a -r option
         setRecursive(true);
       }
   
       // commands operating on local paths have no need for glob expansion
       @Override
       protected List expandArgument(String arg) throws IOException {
         try {
           List items = new LinkedList();
           items.add(new PathData(new URI(arg), getConf()));
           return items;
         } catch (URISyntaxException e) {
           throw new IOException("unexpected URISyntaxException", e);
         }
       }
   
       @Override
       protected void processArguments(LinkedList args)
       throws IOException {
         // NOTE: this logic should be better, mimics previous implementation
         if (args.size() == 1 && args.get(0).toString().equals("-")) {
           copyStreamToTarget(System.in, getTargetPath(args.get(0)));
           return;
         }
         super.processArguments(args);
       }

   procesOptions用于解析参数，expandArgument没看明白，processArguments做实际工作，调用copyStreamToTarget完成copy工作
6. copyStreamToTarget创建TargetFileSystem实例targetFs，TargetFileSystem<=FilterFileSystem<=FileSystem

       TargetFileSystem targetFs = new TargetFileSystem(target.fs);
       try {
         PathData tempTarget = target.suffix("._COPYING_");
         targetFs.setWriteChecksum(writeChecksum);						//跟踪代码发现这个函数貌似什么都没做
         targetFs.writeStreamToFile(in, tempTarget);
         targetFs.rename(tempTarget, target);
       } finally {
         targetFs.close(); // last ditch effort to ensure temp file is removed
       }

   
   
7. writeStreamToFile代码

    void writeStreamToFile(InputStream in, PathData target) throws IOException {
      FSDataOutputStream out = null;
      try {
        out = create(target);
        IOUtils.copyBytes(in, out, getConf(), true);
      } finally {
        IOUtils.closeStream(out); // just in case copyBytes didn't
      }
    }