kylin-cube: 使用REST api来build, 自动构建cube

左手的ㄟ右手 2022-02-17 09:58 296阅读 0赞

### 文章目录 ###

*  1，执行cube的相关任务
 *   *  a, build/rebuild cube
     *  b, 获取job的状态
 *  2, 元数据备份、恢复
 *  3, 自动构建cube

<table> 
 <thead> 
  <tr> 
   <th align="left">api</th> 
   <th>返回值</th> 
   <th align="left">描述</th> 
  </tr> 
 </thead> 
 <tbody> 
  <tr> 
   <td align="left">http://ip:7070/kylin/api/models</td> 
   <td>json数组</td> 
   <td align="left">查看所有model元数据</td> 
  </tr> 
  <tr> 
   <td align="left">http://ip:7070/kylin/api/cubes</td> 
   <td>json数组</td> 
   <td align="left">查看所有cube元数据</td> 
  </tr> 
  <tr> 
   <td align="left">http://ip:7070/kylin/api/jobs/xxx</td> 
   <td>json数组</td> 
   <td align="left">查看单个cube提交后的任务</td> 
  </tr> 
 </tbody> 
</table>

获取用户名密码： 被base64加密后的格式：QURNSU46S1lMSU4=

UserName=ADMIN
    Password=KYLIN
    baseUsernamePasswd=`python -c "import base64; print base64.standard_b64encode('$UserName:$Password')"`

# 1，执行cube的相关任务 #

## a, build/rebuild cube ##

curl -X PUT \
    -H "Authorization: Basic $baseUsernamePasswd" \
    -H "Content-Type: application/json" \
    -d '{ "startTime": 0, "endTime": 1388563200000, "buildType": "BUILD" }' \
    http://localhost:7070/kylin/api/cubes/ys_demo/build
    #http://localhost:7070/kylin/api/cubes/ys_demo/rebuild
    
    ##数据返回
    { "uuid":"2eb3c342-daa2-4fab-abb8-c622beb46d9b", ...}

## b, 获取job的状态 ##

curl -X GET \
    -H "Authorization: Basic QURNSU46S1lMSU4=" \
    -H "Content-Type: application/json" \
    http://localhost:7070/kylin/api/jobs/cbfaf44f-c7bb-4b5a-9da4-671f6c84b1bb
    
    ##返回的数据
    { ...,"submitter":"ADMIN","job_status":"RUNNING","progress":53.84615384615385...}

# 2, 元数据备份、恢复 #

[hdfs@node1 kylin]$ metastore.sh
    usage: metastore.sh backup
           metastore.sh fetch DATA
           metastore.sh reset
           metastore.sh refresh-cube-signature
           metastore.sh restore PATH_TO_LOCAL_META
           metastore.sh list RESOURCE_PATH
           metastore.sh cat RESOURCE_PATH
           metastore.sh remove RESOURCE_PATH
           metastore.sh clean [--delete true]
    
    [hdfs@node1 kylin]$ metastore.sh  list /
    Starting list /
    Retrieving hadoop conf dir......
    [/UUID, /acl, /cube, /cube_desc, /cube_statistics, /dict, /execute, /execute_output, /model_desc, /project, /query, /table, /table_exd, /user]
    
    [hdfs@node1 kylin]$ metastore.sh backup
    Starting backup to /opt/kylin/meta_backups/meta_2019_04_19_12_11_46.....
    
    [hdfs@node1 kylin]$ ls meta_backups/
    meta_2019_04_19_10_29_28  meta_2019_04_19_12_11_46
    [hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28/
    acl   cube_desc        dict     execute_output  project  table      user
    cube  cube_statistics  execute  model_desc      query    table_exd  UUID
    
    [hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28/project/
    CDR_ADMISSION_INFO.json  kylin_demo.json  Ys_kylin_Demo.json
    [hdfs@node1 kylin]$ ls meta_backups/meta_2019_04_19_10_29_28/cube
    aa2423.json  aa2.json  kylin.json  kylinspark.json  ys_demo.json

# 3, 自动构建cube #

文档：[http://kylin.apache.org/cn/docs/howto/howto\_build\_cube\_with\_restapi.html][http_kylin.apache.org_cn_docs_howto_howto_build_cube_with_restapi.html]

*  kylin\_cube\_cron.sh

#!/bin/bash
    set -x
    
    error_file=/tmp/kylin_error.log
    cubes=(
       cube1
       cube2
    )
    
    status_check(){ 
            jobid=$1;
            error_count=0; try_count=0
            while true
            do
                  sleep 20
                  let try_count++
                  
                  # look error jobs, resume it!
                  response=$(curl -s -X GET -H 'Authorization: Basic QURNSU46S1lMSU4=' \ "http://192.168.56.6:7070/kylin/api/jobs/$jobid" )
                    
                  echo $response |grep  -oE job_status.* |grep "FINISHED"
                  [ $? -eq 0 ] && break
    
                  echo $response |grep  -oE job_status.* |grep "ERROR"
                  # try 60 times: 1200 second =20min
                  if [ $? -eq 0 ]  && [ $error_count -le 60 ] ;then 
                     	let error_count++
             	 	 	curl -X PUT -H 'Authorization: Basic QURNSU46S1lMSU4=' -H 'Content-Type: application/json' \ 
    						-d '{"buildType":"BUILD"}' \
                   			"http://192.168.56.6:7070/kylin/api/jobs/$jobid/resume"
                  # wait 6 times: 120 second =2 min to finish 
                  elif [ $try_count -ge 6 ] ; then
                        break
                  else
                     	continue
    	     	  fi	     	 
            done
    }
    
    curl_build(){ 
            cubeid=$1
    
            #1,build cmd
            response=`curl -X PUT -H "Authorization: Basic QURNSU46S1lMSU4=" -H 'Content-Type: application/json' -d '{"buildType":"BUILD"}' \ http://192.168.56.6:7070/kylin/api/cubes/${ cubeid}/rebuild`
    
            #2,resume cmd
            jobid=`echo $response |awk -F',' '{print $1}' |grep 'uuid' |awk -F':' '{print $2}' |xargs`
    
            #check if job is error
    		if [ x$jobid != "x" ] ; then
    	         	status_check $jobid
    		else 
    			echo "$(date '%F %T'), error== $cubeid ==> $response" >> $error_file
    		fi
    }
    
    
    for cube in ${cubes[*]}
    do 
    	curl_build $cube
    done

[http_kylin.apache.org_cn_docs_howto_howto_build_cube_with_restapi.html]: http://kylin.apache.org/cn/docs/howto/howto_build_cube_with_restapi.html