HDFS学习笔记

hadoop fs -ls sqooptest/book

加jdbc.mysql的驱动的路径到classpath中

export

CLASSPATH=$CLASSPATH:$JAVA_HOME/lib:$JAVA_HOME/jre/lib:/tmp/mysql-connector-java-5.1.40/mysql-connector-java-5.1.40-bin.jar

创建一个java文件来验证com.mysql.jdbc.driver的有效性： import java.sql.Connection; import java.sql.DriverManager;

public class testmysql { public static void main(String args[]) { String url = \ String driver = \ try{ Class.forName(driver); } catch(Exception e){ System.out.println(\无法加载驱动\

}

try {

Connection con

DriverManager.getConnection(url,\ if(!con.isClosed()) System.out.println(\ } catch (Exception e) { e.printStackTrace(); } } }

=

Flume实验

在集群中添加 server

它说我已经装了Flume服务

修改elephant与monkey的flume agent配置文件 （monkey）二级agent：

collector1.sources = src1 collector1.channels = ch1 collector1.sinks = sink1

collector1.sources.src1.type = avro collector1.sources.src1.bind = monkey collector1.sources.src1.port = 6000

collector1.sources.src1.channels = ch1

collector1.channels.ch1.type = memory collector1.channels.ch1.capacity = 500

collector1.sinks.sink1.type = hdfs

collector1.sinks.sink1.hdfs.path = /user/flume/collector1 collector1.sinks.sink1.hdfs.filePrefix = access_log collector1.sinks.sink1.channel = ch1

（elephant）一级agent：

tail1.sources = src1 tail1.channels = ch1 tail1.sinks = sink1

tail1.sources.src1.type = exec

tail1.sources.src1.command = tail -F /tmp/access_log tail1.sources.src1.channels = ch1

tail1.channels.ch1.type = memory tail1.channels.ch1.capacity = 500

tail1.sinks.sink1.type = avro

tail1.sinks.sink1.hostname = monkey tail1.sinks.sink1.port = 6000 tail1.sinks.sink1.batch-size = 1 tail1.sinks.sink1.channel = ch1

sudo -u hdfs hdfs dfs -mkdir -p /user/flume/collector1 sudo -u hdfs hdfs dfs -chown -R flume /user/flume

启日志生成器：

./accesslog-gen.bash /tmp/access_log

monkey节点flume agent start

elephant节点flume agent start