目的:

学习用java进行的常用hdfs操作

参考：

环境：

hadoop2.6.4

win7 下的eclipse环境调试已经配置好，参考前面的文章

代码：

1. 创建文件夹

1 package hdfs; 2  3 import java.io.IOException; 4  5 import org.apache.hadoop.conf.Configuration; 6 import org.apache.hadoop.fs.FileSystem; 7 import org.apache.hadoop.fs.Path; 8  9 /**10  * 11  * @author Administrator12  *  创建文件夹,如果不存在13  */14 public class CreateFolder {15     16     public static void main(String[] args) throws IOException {17         Configuration conf =new Configuration();18         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");19         FileSystem fs =  FileSystem.get(conf) ;20         Path path = new Path("/output");21         22         if(! fs.exists(path)){23             fs.mkdirs(path);24         }25     }26 }

以流的方式下载文件

1 package hdfs; 2  3 import java.io.FileNotFoundException; 4 import java.io.FileOutputStream; 5 import java.io.IOException; 6  7 import org.apache.commons.compress.utils.IOUtils; 8 import org.apache.hadoop.conf.Configuration; 9 import org.apache.hadoop.fs.FSDataInputStream;10 import org.apache.hadoop.fs.FileSystem;11 import org.apache.hadoop.fs.Path;12 13 /**14  *  功能：      将 hdfs://ssmaster:9000/data/paper.txt下载到Windows下c:\paper.txt15  *  调用方式：windows下执行,eclipse中执行16  */17 18 public class Down_Load {19 20     public static void main(String[] args) {21         22     Configuration conf =new Configuration();23     conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");24     25     FileSystem fs = null;26     Path src = null;27     FSDataInputStream in = null;28     FileOutputStream out = null;29       30     src = new Path("hdfs://ssmaster:9000/data/paper.txt" );31     32     try {33         34       fs = FileSystem.get(conf) ;35       in = fs.open(src);36 37        } catch (IOException e) {38         e.printStackTrace(); 39     }40     41     try {42         out = new FileOutputStream ("c:\\paper.txt"); //等效  c:/paper.txt43     } catch (FileNotFoundException e) {44         e.printStackTrace();45     }46     47     try {48         IOUtils.copy(in, out);49     } catch (IOException e) {50         e.printStackTrace();51     }52 53 }54 }

View Code

2 上传文件

1 package hdfs; 2  3 import java.io.IOException; 4  5 import org.apache.hadoop.conf.Configuration; 6 import org.apache.hadoop.fs.FileSystem; 7 import org.apache.hadoop.fs.Path; 8  9 /**10  * 11  * @author Administrator12  *  上传本地文件13  */14 public class UploadFile {15     16     public static void main(String[] args) throws IOException {17         Configuration conf =new Configuration();18         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");19         FileSystem fs =  FileSystem.get(conf) ;20         Path path = new Path("/output");21         Path src  = new Path("c:/paper.txt");22         23         fs.copyFromLocalFile(false, true, src, path);24 25     }26 }

3 下载文件

1 package hdfs; 2  3 import java.io.IOException; 4  5 import org.apache.hadoop.conf.Configuration; 6 import org.apache.hadoop.fs.FileSystem; 7 import org.apache.hadoop.fs.Path; 8  9 /**10  * 11  * @author Administrator12  *  上传本地文件13  */14 public class DownFile {15     16     public static void main(String[] args) throws IOException {17         Configuration conf =new Configuration();18         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");19         FileSystem fs =  FileSystem.get(conf) ;20         Path hdfs = new Path("/output/paper.txt");21         Path win7  = new Path("c:/paper_download.txt");22         23         fs.copyToLocalFile(hdfs, win7);24 25     }26 }

4 删除文件

1 package hdfs; 2  3 import java.io.IOException; 4  5 import org.apache.hadoop.conf.Configuration; 6 import org.apache.hadoop.fs.FileSystem; 7 import org.apache.hadoop.fs.Path; 8  9 /**10  * 11  * @author Administrator12  *  删除hdfs文件,如何文件不存在，也运行正常13  */14 public class DeleteFile {15     16     public static void main(String[] args) throws IOException {17         Configuration conf =new Configuration();18         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");19         FileSystem fs =  FileSystem.get(conf) ;20         Path hdfs = new Path("/output/paper.txt");21         fs.delete(hdfs, true);22         23     }24 }

5 显示目录信息

1 package hdfs; 2  3 import java.io.IOException; 4  5 import org.apache.hadoop.conf.Configuration; 6 import org.apache.hadoop.fs.FileStatus; 7 import org.apache.hadoop.fs.FileSystem; 8 import org.apache.hadoop.fs.Path; 9 10 /**11  * 12  * @author Administrator13  *  显示某个目录下的文件14  */15 public class ListFiles {16     17     public static void main(String[] args) throws IOException {18         Configuration conf =new Configuration();19         conf.set("fs.defaultFS", "hdfs://ssmaster:9000/");20         FileSystem fs =  FileSystem.get(conf) ;21         Path hdfs = new Path("/");22         23         24         FileStatus [] files = fs.listStatus(hdfs);25         for (FileStatus file:files) {26           27            System.out.print(file.getPath().getName());28            System.out.print("\t"+ file.isDirectory());29            System.out.print("\t"+ file.getOwner());           30            System.out.print("\n");31         }32 33     }34 }