部署服务端
1：下载hadoop-1.0.0.tar.gz
2：安装JDK 1.6
3：修改配置文件
core-site.xml
fs.default.name hdfs://10.53.132.52:9000
hadoop-env.sh
export JAVA_HOME=/usr/java/jdk1.6.0_29
hdfs-site.xml
dfs.replication 1 dfs.permissions false
mapred-site.xml
mapred.job.tracker 10.53.132.52:9001
masters，slaves 里面设置本机IP（不能为localhost）
赋予本机账户的ssh权限
$ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
4：开启
第一次运行要格式化文件系统，以后就不需要了。
$ bin/hadoop namenode -format
开启
$ bin/start-all.sh
验证
[root@linux52 ~]# jps 20255 NameNode 20565 SecondaryNameNode 20824 TaskTracker 20397 DataNode 20677 JobTracker 24738 Jps [root@linux52 ~]#
调试java api
1：需要的库
1：JDK 1.6 2：commons-lang-2.2.jar 3：commons-configuration-1.6.jar 4：hadoop-core-1.0.0.jar 5：commons-logging-1.1.1.jar
2：代码
public class HDFSUtil { private static Logs log = new Logs(); ; public synchronized static FileSystem getFileSystem(String ip, int port) { FileSystem fs = null; String url = "hdfs://" + ip + ":" + String.valueOf(port); Configuration config = new Configuration(); config.set("fs.default.name", url); try { fs = FileSystem.get(config); } catch (Exception e) { } return fs; } public synchronized static void listNode(FileSystem fs) { DistributedFileSystem dfs = (DistributedFileSystem) fs; try { DatanodeInfo[] infos = dfs.getDataNodeStats(); for (DatanodeInfo node : infos) { System.out.println("HostName: " + node.getHostName() + "/n" + node.getDatanodeReport()); System.out.println("--------------------------------"); } } catch (Exception e) { } } /** * 打印系统配置 * * @param fs */ public synchronized static void listConfig(FileSystem fs) { Iterator > entrys = fs.getConf().iterator(); while (entrys.hasNext()) { Entry item = entrys.next(); log.info(item.getKey() + ": " + item.getValue()); } } /** * 创建目录和父目录 * * @param fs * @param dirName */ public synchronized static void mkdirs(FileSystem fs, String dirName) { // Path home = fs.getHomeDirectory(); Path workDir = fs.getWorkingDirectory(); // String dir = workDir + "/" + dirName;//以前的写法 create directory hdfs://10.53.132.52:9000/user/guoyanwei//tmp/testdir String dir ="hdfs://10.53.132.52:9000" + dirName; Path src = new Path(dir); // FsPermission p = FsPermission.getDefault(); boolean succ; try { succ = fs.mkdirs(src); if (succ) { log.info("create directory " + dir + " successed. "); } else { log.info("create directory " + dir + " failed. "); } } catch (Exception e) { log.error("create directory " + dir + " failed :"+e ); } } /** * 删除目录和子目录 * * @param fs * @param dirName */ public synchronized static void rmdirs(FileSystem fs, String dirName) { // Path home = fs.getHomeDirectory(); // Path workDir = fs.getWorkingDirectory(); String dir = "hdfs://10.53.132.52:9000" + "/" + dirName; Path src = new Path(dir); boolean succ; try { succ = fs.delete(src, true); if (succ) { log.info("remove directory " + dir + " successed. "); } else { log.info("remove directory " + dir + " failed. "); } } catch (Exception e) { log.error("remove directory " + dir + " failed :" ); } } /** * 上传目录或文件 * * @param fs * @param local * @param remote */ public synchronized static void upload(FileSystem fs, String local, String remote) { // Path home = fs.getHomeDirectory(); // Path workDir = fs.getWorkingDirectory(); String dir ="hdfs://10.53.132.52:9000" + remote; Path dst = new Path(dir); Path src = new Path(local); try { fs.copyFromLocalFile(false, true, src, dst); log.info("upload " + local + " to " + remote + " successed. "); } catch (Exception e) { log.error("upload " + local + " to " + remote + " failed :" ); } } /** * 下载目录或文件 * * @param fs * @param local * @param remote */ public synchronized static void download(FileSystem fs, String local, String remote) { Path dst = new Path("hdfs://10.53.132.52:9000/" + remote); Path src = new Path(local); try { fs.copyToLocalFile(false, dst, src); log.info("download from " + remote + " to " + local + " successed. "); } catch (Exception e) { log.error("download from " + remote + " to " + local + " failed :" ); } } /** * 字节数转换 * * @param size * @return */ public synchronized static String convertSize(long size) { String result = String.valueOf(size); if (size < 1024 * 1024) { result = String.valueOf(size / 1024) + " KB"; } else if (size >= 1024 * 1024 && size < 1024 * 1024 * 1024) { result = String.valueOf(size / 1024 / 1024) + " MB"; } else if (size >= 1024 * 1024 * 1024) { result = String.valueOf(size / 1024 / 1024 / 1024) + " GB"; } else { result = result + " B"; } return result; } /** * 遍历HDFS上的文件和目录 * * @param fs * @param path */ public synchronized static void listFile(FileSystem fs, String path) { Path dst = new Path("hdfs://10.53.132.52:9000/" + path); try { String relativePath = ""; FileStatus[] fList = fs.listStatus(dst); for (FileStatus f : fList) { if (null != f) { relativePath = new StringBuffer() .append(f.getPath().getParent()).append("/") .append(f.getPath().getName()).toString(); if (f.isDir()) { listFile(fs, relativePath); } else { System.out.println(convertSize(f.getLen()) + "/t/t" + relativePath); } } } } catch (Exception e) { } finally { } } public synchronized static void write(FileSystem fs, String path, String data) { Path dst = new Path("hdfs://10.53.132.52:9000/" + path); try { FSDataOutputStream dos = fs.create(dst); dos.writeUTF(data); dos.close(); log.info("write content to " + path + " successed. "); } catch (Exception e) { } } public synchronized static String read(FileSystem fs, String path) { String content = null; Path dst = new Path("hdfs://10.53.132.52:9000/" + path); try { // reading FSDataInputStream dis = fs.open(dst); content = dis.readUTF(); dis.close(); log.info("read content from " + path + " successed. "); } catch (Exception e) { } return content; } 3：调用
public static void main(String[] args) throws Exception { FileSystem fs = null; try { fs = HDFSUtil.getFileSystem("10.53.132.52", 9000); String dirName = "/tmp/test/aa"; String localSrc = "d://8.log"; String dst = "/tmp/8.log"; //上传本地文件 //HDFSUtil.upload(fs, localSrc ,dst); //HDFSUtil.listNode(fs); //创建目录 //HDFSUtil.mkdirs(fs, dirName); //删除目录 //HDFSUtil.rmdirs(fs, dirName); //下载文件到本地 //HDFSUtil.download(fs, localSrc, dst); //创建文件 //HDFSUtil.write(fs, "/tmp/9.log", "test-测试"); //读取文件 //String content = HDFSUtil.read(fs, "/tmp/9.log"); //System.out.println(content); //遍历文件夹 //HDFSUtil.listFile(fs, "/tmp/"); //遍历节点 //HDFSUtil.listNode(fs); //遍历配置信息 //HDFSUtil.listConfig(fs); } catch (Exception e) { // TODO: handle exception } finally { if (fs != null) { fs.close(); } } System.out.println("over"); } 4：管理地址
http://10.53.132.52:50070 http://10.53.132.52:50030
执行结果
后记
1：小文件系统需要研究 TFS
2：不同语言调用需要研究 Avro