Presto和Alluxio结合使用

以你之姓@ 2021-09-28 23:38 598阅读 0赞

# 1.下载presto #

[https://prestodb.io/docs/current/installation/deployment.html][https_prestodb.io_docs_current_installation_deployment.html]

presto-server-0.215.tar.gz

# 2.解压 #

cd /opt/presto
    tar xvf presto-server-0.215.tar.gz

# 3.配置 #

cd presto-server-0.215
    mkdir  etc/

新建 etc/node.propertie

vi etc/node.propertie
    内容如下：
    node.environment=production
    node.id=ffffffff-ffff-ffff-ffff-ffffffffffff
    node.data-dir=/var/presto/data

新建 etc/jvm.config

-server
    -Xmx16G
    -XX:+UseG1GC
    -XX:G1HeapRegionSize=32M
    -XX:+UseGCOverheadLimit
    -XX:+ExplicitGCInvokesConcurrent
    -XX:+HeapDumpOnOutOfMemoryError
    -XX:+ExitOnOutOfMemoryError
    -Xbootclasspath/p:/opt/alluxio-1.8.1/conf

新建 etc/config.properties

coordinator=true
    node-scheduler.include-coordinator=true
    http-server.http.port=8080
    query.max-memory=5GB
    query.max-memory-per-node=1GB
    query.max-total-memory-per-node=2GB
    discovery-server.enabled=true
    discovery.uri=http://192.168.6.12:8080

新建 etc/log.properties

com.facebook.presto=INFO

mkdir etc/catalog
    vi etc/catalog/jmx.properties
    connector.name=jmx

# 4.启动 #

bin/launcher start

访问地址

[http://192.168.6.12:8080/ui/][http_192.168.6.12_8080_ui]

# Presto集成alluxio配置 #

## 1.增加presto配置文件 ##

vi/catalog/hive.properties

内容如下：

connector.name=hive-hadoop2 hive.metastore.uri=thrift://192.168.6.12:9083

## 2.复制客户端 ##

复制Alluxio client jar /<PATH\_TO\_ALLUXIO>/client/alluxio-1.8.1-client.jar 到presto中$\{PRESTO\_HOME\}/plugin/hive-hadoop2/

cp /opt/alluxio-1.8.1/client/alluxio-1.8.1-client.jar plugin/hive-hadoop2/

## 3.重启presto ##

bin/launcher restart

## 4.测试 ##

进入hive命令行建表：

CREATE TABLE u_user (
    userid INT,
    age INT,
    gender CHAR(1),
    occupation STRING,
    zipcode STRING)
    ROW FORMAT DELIMITED
    FIELDS TERMINATED BY '|'
    LOCATION 'alluxio://192.168.6.12:19998/ml-100k';
    
    create  EXTERNAL table rating_alluxio(
    userId INT,movieId INT,rating FLOAT,timestamps STRING
    )row format delimited 
    fields terminated by ',' 
    LOCATION 'alluxio://192.168.6.12:19998/ml-latest';
    
    create  EXTERNAL table rating_hdfs(
    userId INT,movieId INT,rating FLOAT,timestamps STRING
    )row format delimited 
    fields terminated by ','
    LOCATION '/ml-latest';

## 5.启动 Hive metastore ##

hive --service metastore &

## 6.下载presto-cli-0.215-executable.jar ##

scp presto-cli-0.215-executable.jar root@192.168.6.12:/opt/presto
    
    cp /opt/presto/presto-cli-0.215-executable.jar bin/presto
    
    chmod +x bin/presto

## 7.启动presto客户端 ##

bin/presto --server 192.168.6.12:8080 --catalog hive --schema default

# 测试查询语句： #

select count(*) from rating_alluxio;
    
    select count(*) from rating_hdfs;

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3YWh1dA_size_16_color_FFFFFF_t_70][]

select avg(rating) from rating_alluxio;
    select avg(rating) from rating_hdfs;

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3YWh1dA_size_16_color_FFFFFF_t_70 1][]

select sum(rating) from rating_alluxio;
    select sum(rating) from rating_hdfs;

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3YWh1dA_size_16_color_FFFFFF_t_70 2][]

遇到问题：

![2019061110430153.png][]

更新jdk版本

下载jdk最新版本

jdk-8u191-linux-x64.rpm

scp jdk-8u191-linux-x64.rpm root@192.168.6.12:/opt/presto

增加执行权限

chmod +x jdk-8u191-linux-x64.rpm

安装jdk

rpm -ivh jdk-8u191-linux-x64.rpm

mv /usr/java/jdk1.8.0\_191-amd64/ /usr/java/jdk1.8.0\_191

修改环境变量

vi /etc/bashrc

export JAVA\_HOME=/usr/java/jdk1.8.0\_191

使环境变量生效

source /etc/bashrc

![201906111043351.png][]

在presto配置文件etc/jmx.config增加：

\-Xbootclasspath/p:/opt/alluxio-1.8.1/conf

[https_prestodb.io_docs_current_installation_deployment.html]: https://prestodb.io/docs/current/installation/deployment.html
[http_192.168.6.12_8080_ui]: http://192.168.6.12:8080/ui/
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3YWh1dA_size_16_color_FFFFFF_t_70]: /images/20210923/1bf52e52729841079cb1461d35214417.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3YWh1dA_size_16_color_FFFFFF_t_70 1]: /images/20210923/e14c995fc19b41c4808f1c5c52472ada.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3p3YWh1dA_size_16_color_FFFFFF_t_70 2]: /images/20210923/6fbbd4b51b0a40eb8bae5d09da42a45e.png
[2019061110430153.png]: /images/20210923/14047303030649149ac636944f2ad7c6.png
[201906111043351.png]: /images/20210923/c61da4fdc86041968a5f995bddfad5e4.png