mkdir -p /data/software

cd /data/software

wget https://github.com/InterestingLab/waterdrop/releases/download/v1.1.2/waterdrop-1.1.2.zip -O waterdrop-1.1.2.zip

unzip waterdrop-1.1.2.zip

ln -s waterdrop-1.1.2 waterdrop

# 修改sparkHome路径

cd waterdrop

vim config/waterdrop-env.sh

# SPARK_HOME=${SPARK_HOME:-/opt/spark}

SPARK_HOME=/opt/cloudera/parcels/SPARK2/lib/spark2

# 测试 hive to clickHouse

cp config/batch.conf.template config/batch.conf

vim config/batch.conf

# 配置文件的内容

spark {

spark.app.name = "Waterdrop"

spark.executor.instances = 2

spark.executor.cores = 1

spark.executor.memory = "1g"

}

input {

hive {

pre_sql = "select * from access.nginx_msg_detail"

table_name = "access_log"

}

}

filter {

remove {

source_field = ["minute", "hour"]

}

}

output {

clickhouse {

host = "your.clickhouse.host:8123"

database = "waterdrop"

table = "access_log"

fields = ["date", "datetime", "hostname", "uri", "http_code", "request_time", "data_size", "domain"]

username = "username"

password = "password"

}

}