大数据分析配置文件

男娘i 2022-07-12 03:50 217阅读 0赞

logstash.sh

\#!/bin/bash

/home/haoren/data/logstash-5.0.2/bin/logstash -f  lkbillserver.conf &

lkbillserver.conf配置如下

input \{

file \{

path => "/log/lkbillserver.log"

codec => plain \{

charset => "GBK"

\#start\_position => "beginning"

\#sincedb\_path => "/dev/null"

type => "lkbillserver"

filter \{

if (\[message\] =~ "人民币统计")\{

mutate \{replace => \{ "type" => "dubijiesuan" \}\}

grok \{

\#170208-10:00:28 Bill\[40268\]  INFO: \[人民币结算\]时间(1486519228),用户身份(30581009),原来人民币(12567),现在人民币(16567),人民币操作(1),增加(4000),操作类型(19),操作详情(2),操作数量(0).描述:通用人民币操作

\#170208-10:00:01 Bill\[40268\]  INFO: \[人民币结算\]时间(1486519201),用户身份(22327945),原来人民币(2572),现在人民币(2532),人民币操作(2),扣除(40),操作类型(19),操作详情(15),操作数量(0).描述:通用人民币操作

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) Bill\\\[\\d+\\\]  INFO: \\\[人民币结算\\\]时间\$%\{NUMBER:time:int\}\$,用户身份\$%\{NUMBER:user身份:int\}\$,原来人民币\$%\{NUMBER:oldcoin:int\}\$,现在人民币\$%\{NUMBER:currentcoin:int\}\$,人民币操作\$%\{NUMBER:coinop:int\}\$,%\{DATA:sop\}\$%\{NUMBER:coinnum:int\}\$,操作类型\$%\{NUMBER:optype:int\}\$,操作详情\$%\{NUMBER:opdetail:int\}\$,操作数量\$%\{NUMBER:opnum:int\}\$.描述:%\{DATA:sinfo\}"\]

else\{

drop\{\}

date \{

match => \["datetime", "yyMMdd-HH:mm:ss"\]

\#timezone => \["Asia/Hong\_Kong"\]

\#remove\_field => \["time"\]

ruby \{

code => "event.timestamp.time.localtime"

mutate \{

\#some pc no host

replace => \{ "host" => "192.168.10.7" \}

output \{

\#stdout \{

\#    codec => plain \{

\#        charset => "UTF-8"

\#        \#charset => "GBK"

\#    \}

\#\}

\#file \{

\#    path => "/tmp/logstash.log"

\#    codec => json \{

\#        charset => "UTF-8"

\#    \}

\#\}

redis \{

host => \["192.168.10.8"\]

port => 6379

data\_type => "list"

key => "logstash"

password => "A8841c09BAD52E63067C4DA"

codec => json \{

charset => "UTF-8"

/home/haoren/data/elasticsearch-5.0.2/configelasticsearch.yml配置如下

cluster.name: es-5.0-haoren

node.name: node-18

network.host: 0.0.0.0

http.port: 9200

http.cors.enabled: true

http.cors.allow-origin: "\*"

elasticsearch.sh

\#!/bin/bash

cd /home/haoren/data/elasticsearch-5.0.2/

/home/haoren/data/elasticsearch-5.0.2/bin/elasticsearch -d

/home/haoren/data/kibana-5.0.2-linux-x86\_64/configkibana.yml配置如下

server.port: 5601

server.host: "0.0.0.0"

elasticsearch.url: "http://53.92.155.26:9200"

kibana.sh

\#!/bin/bash

cd /home/haoren/data/kibana-5.0.2-linux-x86\_64

/home/haoren/data/kibana-5.0.2-linux-x86\_64/bin/kibana serve &

logstash.sh

\#!/bin/bash

/home/haoren/data/logstash-5.0.2/bin/logstash -f logstash.conf &

/home/haoren/data/logstash-5.0.2/logstash.conf配置如下

input \{

redis \{

host => \["192.168.10.8"\]

port => 6379

password => "A8841c09BAD52E63067C4DA"

data\_type => "list"

key => "logstash"

codec => json \{

charset => "UTF-8"

filter \{

ruby \{

code=>"event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"

output \{

elasticsearch \{

hosts => \["127.0.0.1:9200"\]

index => "%\{type\}-%\{daytag\}"

user => "logstashserver"

password => "A950C0FB2D833E42C1AC59210CD5CDF8"

/home/haoren/data/logstash-5.0.2/filebeat.sh

\#!/bin/bash

/home/haoren/data/logstash-5.0.2/bin/logstash -f filebeat.conf &

/home/haoren/data/logstash-5.0.2/filebeat.conf配置如下

input \{

redis \{

host => \["192.168.10.8"\]

port => 26379

password => "A8841c09BAD52E63067C4DA"

data\_type => "list"

key => "filebeat"

codec => json \{

charset => "UTF-8"

\#file \{

\#    path => "/home/haoren/data/logstash-5.0.2/filebeat.txt"

\#    codec => plain \{

\#        charset => "GBK"

\#    \}

\#    start\_position => "beginning"

\#    sincedb\_path => "/dev/null"

\#    type => "pchannelserver"

\#\}

\#file \{

\#   path => "/home/haoren/data/logstash-5.0.2/filebeat.json"

\#   codec => json \{

\#       charset => "UTF-8"

\#   \}

\#   start\_position => "beginning"

\#   sincedb\_path => "/dev/null"

\#   type => "pchannelserver"

\#\}

filter \{

if( \[type\] == "sessionserver" )\{

if( \[message\] =~ "登陆统计")\{

mutate \{replace => \{ "type" => "userlogin" \}\}

grok \{

\#161206-16:00:00 SS\[4306\] TRACE: \[登陆统计\]收到角色(82559870)登陆(PC)IP(124.239.95.209)MAC(52229449286)机器身份(454070640)渠道(0)game身份(6)端口(3889)登陆类型(1)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) SS\\\[\\d+\\\].\*?TRACE: \\\[登陆统计\\\]收到角色\$%\{NUMBER:user身份:int\}\$登陆\$%\{DATA:sclient\}\$IP\$%\{DATA:sip\}\$MAC\$%\{DATA:smac\}\$机器身份\$%\{NUMBER:m身份:int\}\$渠道\$%\{NUMBER:apk身份:int\}\$game身份\$%\{NUMBER:game身份:int\}\$端口\$%\{NUMBER:port:int\}\$登陆类型\$%\{NUMBER:logintype:int\}\$"\]

else if (\[message\] =~ "注册统计")\{

mutate \{replace => \{ "type" => "userreg" \}\}

grok\{

\#161205-15:33:22 SS\[4306\] TRACE: \[注册统计\]用户(87475178)注册(Andro身份)渠道(8)账号(\_wx\_omj\_avq\_nz8obsk6yy5dsmfrlfmk)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) SS\\\[\\d+\\\].\*?TRACE: \\\[注册统计\\\]用户\$%\{NUMBER:user身份:int\}\$注册\$%\{DATA:sclient\}\$渠道\$%\{NUMBER:apk身份:int\}\$账号\$%\{DATA:saccount\}\$" \]

else if( \[type\] == "activityserver" )\{

if( \[message\] =~ "用户注册渠道奖励统计")\{

mutate \{replace => \{ "type" => "actvityregreward" \}\}

grok \{

\#161212-10:17:29 ActivityServer\[17702\]  INFO: \[UserRegisterReward.cpp:90\] \[用户注册渠道奖励统计\]用户(87582819)客户端(0)注册渠道(1001)机器码(476931654)奖励包裹(77)个数(100)当日奖励(100)上限(1000000)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) ActivityServer\\\[\\d+\\\].\*?INFO: .\*?\\\[用户注册渠道奖励统计\\\]用户\$%\{NUMBER:user身份:int\}\$客户端\$%\{NUMBER:client:int\}\$注册渠道\$%\{NUMBER:apk身份:int\}\$机器码\$%\{NUMBER:m身份:int\}\$奖励包裹\$%\{NUMBER:pack身份:int\}\$个数\$%\{NUMBER:num:int\}\$当日奖励\$%\{NUMBER:todaynum:int\}\$上限\$%\{NUMBER:maxnum:int\}\$\\)"\]

else if( \[message\] =~ "充值礼包统计")\{

mutate \{replace => \{ "type" => "activityrechargebag" \}\}

grok \{

\#161220-17:02:53 ActivityServer\[17700\]  INFO: \[ActivityRechargeBag.cpp:559\] \[充值礼包统计\]用户(60163778)获得(8800)礼包类型(2)用户获得( 1029:1)播获得( 1062:688)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) ActivityServer\\\[\\d+\\\].\*?INFO: \\\[.\*?\\\] \\\[充值礼包统计\\\]用户\$%\{NUMBER:user身份:int\}\$获得\$%\{NUMBER:p身份:int\}\$礼包类型\$%\{NUMBER:ptype:int\}\$用户获得\$%\{DATA:userget\}\$官方获得\$%\{DATA:singerget\}\$"\]

else if( \[message\] =~ "盛典分数统计")\{

mutate \{replace => \{ "type" => "activitymodules" \}\}

grok \{

\#161220-17:05:31 ActivityServer\[17702\]  INFO: \[ActivityModules.cpp:837\]\[竞技赛\] \[盛典分数统计\]用户(84795176)平台(65780103)播(65780103)个数(20)礼物(92)价值(0)原分数(385399)增加分数(20)现分数(385419)淘汰(0)新旧(0)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) ActivityServer\\\[\\d+\\\]  INFO: \\\[.\*?\\\]\\\[2016竞技赛\\\] \\\[盛典分数统计\\\]用户\$%\{NUMBER:user身份:int\}\$平台\$%\{NUMBER:channel身份:int\}\$播\$%\{NUMBER:singer身份:int\}\$个数\$%\{NUMBER:num:int\}\$礼物\$%\{NUMBER:item身份:int\}\$价值\$%\{NUMBER:coin:int\}\$原分数\$%\{NUMBER:oldscore:int\}\$增加分数\$%\{NUMBER:addscore:int\}\$现分数\$%\{NUMBER:nowscore:int\}\$淘汰\$%\{NUMBER:out:int\}\$新旧\$%\{NUMBER:isnew:int\}\$"\]

else if( \[type\] == "vchannelserver" )\{

if( \[message\] =~ "进出平台统计")\{

mutate \{replace => \{ "type" => "vchannelin" \}\}

grok \{

\#170109-12:59:39 VChannelServer\[15000\]  INFO: \[Channel.cpp:414\] \[进出平台统计\]用户(22016998)(进入)平台(3998186)端(0)渠道(0)IMState(6)机器身份(246418457)播(87261227)游客(0)

\#170109-12:59:40 VChannelServer\[15000\]  INFO: \[Channel.cpp:414\] \[进出平台统计\]用户(83735196)(离开)平台(4029779)端(3)渠道(0)IMState(13)机器身份(0)播(0)游客(0)

\#170118-09:59:58 VChannelServer\[15001\]  INFO: \[Channel.cpp:417\] \[进出平台统计\]用户(87455625)(进入)平台(3830989)端(3)渠道(2017)IMState(13)机器身份(0)播(65256549)游客(0)身份(0)

\#170118-09:59:59 VChannelServer\[15001\]  INFO: \[Channel.cpp:417\] \[进出平台统计\]用户(3015482583)(离开)平台(3830989)端(0)渠道(0)IMState(13)机器身份(0)播(65256549)游客(1)游戏身份(0)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) VChannelServer\\\[\\d+\\\]  INFO: \\\[.\*?\\\] \\\[进出平台统计\\\]用户\$%\{NUMBER:user身份:int\}\$\$%\{DATA:sop\}\$平台\$%\{NUMBER:channel身份:int\}\$端\$%\{NUMBER:client:int\}\$渠道\$%\{NUMBER:apk身份:int\}\$IMState\$%\{NUMBER:imstate:int\}\$机器身份\$%\{NUMBER:m身份:int\}\$播\$%\{NUMBER:singer身份:int\}\$客\$%\{NUMBER:istemp:int\}\$游戏身份\$%\{NUMBER:game身份:int\}\$"\]

else if( \[type\] == "pchannelserver" )\{

if( \[message\] =~ "进出平台统计")\{

mutate \{replace => \{ "type" => "pchannelin" \}\}

grok \{

\#170111-00:00:17 PChannelServer\[18701\]  INFO: \[进出平台统计\]用户(88464974)(进入)平台(86972527)端(0)渠道(0)IMState(6)机器身份(0)播(86972527)客(0)

\#170111-00:00:43 PChannelServer\[18701\]  INFO: \[进出平台统计\]用户(88464904)(退出)平台(86972527)端(0)渠道(0)IMState(6)机器身份(0)播(86972527)客(0)

match => \[ "message", "(?(?>\\d\{6\}-\\d\\d:\\d\\d:\\d\\d)) PChannelServer\\\[\\d+\\\]  INFO: \\\[进出平台统计\\\]用户\$%\{NUMBER:user身份:int\}\$\$%\{DATA:sop\}\$平台\$%\{NUMBER:channel身份:int\}\$端\$%\{NUMBER:client:int\}\$渠道\$%\{NUMBER:apk身份:int\}\$IMState\$%\{NUMBER:imstate:int\}\$机器身份\$%\{NUMBER:m身份:int\}\$播\$%\{NUMBER:singer身份:int\}\$游客\$%\{NUMBER:istemp:int\}\$"\]

else if( \[type\] == "billserver" )\{

if( \[message\] =~ "人民币统计")\{

mutate \{replace => \{ "type" => "dubijiesuan" \}\}

grok \{

else\{

drop\{\}

date \{

match => \["datetime", "yyMMdd-HH:mm:ss"\]

ruby \{

code => "event.timestamp.time.localtime"

ruby \{

code => "event.set('daytag',event.timestamp.time.localtime.strftime('%Y.%m.%d'))"

remove\_field => \["tags"\]

output \{

\#stdout \{

\#    codec => plain \{

\#        charset => "UTF-8"

\#        \#charset => "GBK"

\#    \}

\#\}

\#file \{

\#    path => "/tmp/logstash.log"

\#    codec => json \{

\#        charset => "UTF-8"

\#    \}

\#\}

elasticsearch \{

hosts => \["127.0.0.1:19200"\]

index => "%\{type\}-%\{daytag\}"

\#index => "%\{type\}-%\{+yyyy.MM.dd\}"

user => "logstashserver"

password => "A950C0FB2D833E42C1AC59210CD5CDF8"

/home/haoren/soft/zeppelin-0.6.2-bin-all/conf/shiro.ini 配置如下

\[users\]

admin = haoren@123.com\_z

user1 = password2, role1, role2

user2 = password3, role3

user3 = password4, role2

hao1 = hao1@haoren.123, role1

zhang1 = zhang@haoren123, role1

yu = yu@haoren124

lian = lian@haoren333

\[main\]

sessionManager = org.apache.shiro.web.session.mgt.DefaultWebSessionManager

securityManager.sessionManager = $sessionManager

securityManager.sessionManager.globalSessionTimeout = 86400000

shiro.loginUrl = /api/login

\[roles\]

role1 = \*

role2 = \*

role3 = \*

\[urls\]

/api/version = anon

/\*\* = authc

/home/haoren/soft/zeppelin-0.6.2-bin-all/conf/zeppelin-env.sh配置如下

export  SPARK\_MASTER\_IP=127.0.0.1

export  SPARK\_LOCAL\_IP=127.0.0.1

export ZEPPELIN\_MEM="-Xms1024m -Xmx16384m -XX:MaxPermSize=16384m"

/home/haoren/soft/zeppelin-0.6.2-bin-all/conf/zeppelin-site.xml配置如下

zeppelin.server.port

18081

Server port.

/home/haoren/soft/zeppelin-0.6.2-bin-all/conf/notebook-authorization.json配置如下

"authInfo": \{

"2C319P4S7": \{

"owners": \[

"hao"

\],

"readers": \[

"hao"

\],

"writers": \[

"zhang",

"hao",

"yu"

\},

"2C1JVSN53": \{

"owners": \[

"hao"

\],

"readers": \[

"hao"

\],

"writers": \[

"hao"

\},

"2C1M3J3XZ": \{

"owners": \[

"hao"

\],

"readers": \[

"hao"

\],

"writers": \[

"hao"

\},

"2C1XJZXGT": \{

"owners": \[

"hao"

\],

"readers": \[

"hao"

\],

"writers": \[

"zhang",

"hao",

"yu"

\},

"2C56N6X9C": \{

"owners": \[

"hao"

\],

"readers": \[

"hao",

"yu"

\],

"writers": \[

"hao"

\},

"2C49TEC59": \{

"owners": \[

"hao"

\],

"readers": \[

"hao"

\],

"writers": \[

"hao"

\},

"2C6K6NBSS": \{

"owners": \[

"hao"

\],

"readers": \[

"lian"

\],

"writers": \[

"hao",

"lian",

"yu"

/home/haoren/soft/zeppelin-0.6.2-bin-all/conf/interpreter.json配置如下

"interpreterSettings": \{

"2BZ82J2SU": \{

"id": "2BZ82J2SU",

"name": "psql",

"group": "psql",

"properties": \{

"postgresql.password": "",

"postgresql.driver.name": "org.postgresql.Driver",

"postgresql.max.result": "1000",

"postgresql.url": "jdbc:postgresql://localhost:5432/",

"postgresql.user": "gpadmin"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.postgresql.PostgreSqlInterpreter",

"name": "sql"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BY5XFDQC": \{

"id": "2BY5XFDQC",

"name": "flink",

"group": "flink",

"properties": \{

"port": "6123",

"host": "local"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.flink.FlinkInterpreter",

"name": "flink"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BZ3DPPQX": \{

"id": "2BZ3DPPQX",

"name": "jdbc",

"group": "jdbc",

"properties": \{

"phoenix.user": "phoenixuser",

"hive.url": "jdbc:hive2://localhost:10000",

"default.driver": "org.postgresql.Driver",

"phoenix.driver": "org.apache.phoenix.jdbc.PhoenixDriver",

"hive.user": "hive",

"psql.password": "",

"psql.user": "phoenixuser",

"psql.url": "jdbc:postgresql://localhost:5432/",

"default.user": "gpadmin",

"phoenix.hbase.client.retries.number": "1",

"phoenix.url": "jdbc:phoenix:localhost:2181:/hbase-unsecure",

"tajo.url": "jdbc:tajo://localhost:26002/default",

"tajo.driver": "org.apache.tajo.jdbc.TajoDriver",

"psql.driver": "org.postgresql.Driver",

"default.password": "",

"zeppelin.jdbc.concurrent.use": "true",

"hive.password": "",

"hive.driver": "org.apache.hive.jdbc.HiveDriver",

"common.max\_count": "1000",

"phoenix.password": "",

"zeppelin.jdbc.concurrent.max\_connection": "10",

"default.url": "jdbc:postgresql://localhost:5432/"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.jdbc.JDBCInterpreter",

"name": "sql"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BZTZF2EV": \{

"id": "2BZTZF2EV",

"name": "kylin",

"group": "kylin",

"properties": \{

"kylin.api.user": "ADMIN",

"kylin.query.limit": "5000",

"kylin.api.password": "KYLIN",

"kylin.query.offset": "0",

"kylin.query.project": "default",

"kylin.api.url": "http://\\u003chost\\u003e:\\u003cport\\u003e/kylin/api/query",

"kylin.query.ispartial": "true"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.kylin.KylinInterpreter",

"name": "kylin"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2C1DG6ZVR": \{

"id": "2C1DG6ZVR",

"name": "elasticsearch",

"group": "elasticsearch",

"properties": \{

"elasticsearch.result.size": "10",

"elasticsearch.port": "9300",

"elasticsearch.cluster.name": "elasticsearch",

"elasticsearch.host": "localhost"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter",

"name": "elasticsearch"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BYKSJZ9C": \{

"id": "2BYKSJZ9C",

"name": "ignite",

"group": "ignite",

"properties": \{

"ignite.peerClassLoadingEnabled": "true",

"ignite.config.url": "",

"ignite.jdbc.url": "jdbc:ignite:cfg://default-ignite-jdbc.xml",

"ignite.clientMode": "true",

"ignite.addresses": "127.0.0.1:47500..47509"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.ignite.IgniteInterpreter",

"name": "ignite"

\},

"class": "org.apache.zeppelin.ignite.IgniteSqlInterpreter",

"name": "ignitesql"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BYECAAP6": \{

"id": "2BYECAAP6",

"name": "angular",

"group": "angular",

"properties": \{\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.angular.AngularInterpreter",

"name": "angular"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BZJZ266V": \{

"id": "2BZJZ266V",

"name": "file",

"group": "file",

"properties": \{

"hdfs.user": "hdfs",

"hdfs.maxlength": "1000",

"hdfs.url": "http://localhost:50070/webhdfs/v1/"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.file.HDFSFileInterpreter",

"name": "hdfs"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BXY3DBVE": \{

"id": "2BXY3DBVE",

"name": "hbase",

"group": "hbase",

"properties": \{

"zeppelin.hbase.test.mode": "false",

"hbase.ruby.sources": "lib/ruby",

"hbase.home": "/usr/lib/hbase/"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.hbase.HbaseInterpreter",

"name": "hbase"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BY98FHVP": \{

"id": "2BY98FHVP",

"name": "cassandra",

"group": "cassandra",

"properties": \{

"cassandra.query.default.consistency": "ONE",

"cassandra.pooling.max.connection.per.host.local": "8",

"cassandra.load.balancing.policy": "DEFAULT",

"cassandra.pooling.new.connection.threshold.local": "100",

"cassandra.credentials.password": "none",

"cassandra.native.port": "9042",

"cassandra.hosts": "localhost",

"cassandra.pooling.core.connection.per.host.local": "2",

"cassandra.retry.policy": "DEFAULT",

"cassandra.pooling.pool.timeout.millisecs": "5000",

"cassandra.protocol.version": "4",

"cassandra.pooling.new.connection.threshold.remote": "100",

"cassandra.pooling.core.connection.per.host.remote": "1",

"cassandra.pooling.max.request.per.connection.remote": "256",

"cassandra.cluster": "Test Cluster",

"cassandra.pooling.max.request.per.connection.local": "1024",

"cassandra.speculative.execution.policy": "DEFAULT",

"cassandra.compression.protocol": "NONE",

"cassandra.query.default.fetchSize": "5000",

"cassandra.keyspace": "system",

"cassandra.pooling.idle.timeout.seconds": "120",

"cassandra.credentials.username": "none",

"cassandra.max.schema.agreement.wait.second": "10",

"cassandra.socket.tcp.no\_delay": "true",

"cassandra.interpreter.parallelism": "10",

"cassandra.query.default.serial.consistency": "SERIAL",

"cassandra.socket.connection.timeout.millisecs": "5000",

"cassandra.pooling.max.connection.per.host.remote": "2",

"cassandra.pooling.heartbeat.interval.seconds": "30",

"cassandra.reconnection.policy": "DEFAULT",

"cassandra.socket.read.timeout.millisecs": "12000"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.cassandra.CassandraInterpreter",

"name": "cassandra"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2C1SV5WUS": \{

"id": "2C1SV5WUS",

"name": "alluxio",

"group": "alluxio",

"properties": \{

"alluxio.master.port": "19998",

"alluxio.master.hostname": "localhost"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.alluxio.AlluxioInterpreter",

"name": "alluxio"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BZ2H37EF": \{

"id": "2BZ2H37EF",

"name": "bigquery",

"group": "bigquery",

"properties": \{

"zeppelin.bigquery.max\_no\_of\_rows": "100000",

"zeppelin.bigquery.project\_id": " ",

"zeppelin.bigquery.wait\_time": "5000"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.bigquery.BigQueryInterpreter",

"name": "sql"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2C29FZKVG": \{

"id": "2C29FZKVG",

"name": "spark",

"group": "spark",

"properties": \{

"spark.executor.memory": "",

"args": "",

"zeppelin.spark.printREPLOutput": "true",

"spark.cores.max": "",

"zeppelin.dep.additionalRemoteRepository": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",

"zeppelin.spark.sql.stacktrace": "true",

"zeppelin.spark.importImplicit": "true",

"zeppelin.spark.concurrentSQL": "false",

"zeppelin.spark.useHiveContext": "true",

"zeppelin.pyspark.python": "python",

"zeppelin.dep.localrepo": "local-repo",

"zeppelin.interpreter.localRepo": "/home/haoren/soft/zeppelin/local-repo/2C29FZKVG",

"zeppelin.R.knitr": "true",

"zeppelin.spark.maxResult": "1000",

"master": "local\[\*\]",

"spark.app.name": "Zeppelin",

"zeppelin.R.image.width": "100%",

"zeppelin.R.render.options": "out.format \\u003d \\u0027html\\u0027, comment \\u003d NA, echo \\u003d FALSE, results \\u003d \\u0027asis\\u0027, message \\u003d F, warning \\u003d F",

"zeppelin.R.cmd": "R"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.spark.SparkInterpreter",

"name": "spark"

\},

"class": "org.apache.zeppelin.spark.PySparkInterpreter",

"name": "pyspark"

\},

"class": "org.apache.zeppelin.spark.SparkRInterpreter",

"name": "r"

\},

"class": "org.apache.zeppelin.spark.SparkSqlInterpreter",

"name": "sql"

\},

"class": "org.apache.zeppelin.spark.DepInterpreter",

"name": "dep"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BYE767ZH": \{

"id": "2BYE767ZH",

"name": "md",

"group": "md",

"properties": \{

"zeppelin.interpreter.localRepo": "/home/haoren/soft/zeppelin/local-repo/2BYE767ZH"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.markdown.Markdown",

"name": "md"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2BZMQHQTC": \{

"id": "2BZMQHQTC",

"name": "sh",

"group": "sh",

"properties": \{

"shell.command.timeout.millisecs": "60000"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.shell.ShellInterpreter",

"name": "sh"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2C17RTEG8": \{

"id": "2C17RTEG8",

"name": "livy",

"group": "livy",

"properties": \{

"livy.spark.driver.cores": "",

"zeppelin.livy.principal": "",

"zeppelin.livy.keytab": "",

"livy.spark.executor.instances": "",

"livy.spark.driver.memory": "",

"livy.spark.dynamicAllocation.enabled": "",

"zeppelin.livy.concurrentSQL": "false",

"livy.spark.dynamicAllocation.cachedExecutorIdleTimeout": "",

"livy.spark.master": "local\[\*\]",

"zeppelin.livy.spark.sql.maxResult": "1000",

"livy.spark.executor.cores": "",

"livy.spark.executor.memory": "",

"zeppelin.livy.url": "http://localhost:8998",

"livy.spark.dynamicAllocation.initialExecutors": "",

"livy.spark.dynamicAllocation.minExecutors": "",

"livy.spark.dynamicAllocation.maxExecutors": ""

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.livy.LivySparkInterpreter",

"name": "spark"

\},

"class": "org.apache.zeppelin.livy.LivyPySparkInterpreter",

"name": "pyspark"

\},

"class": "org.apache.zeppelin.livy.LivySparkRInterpreter",

"name": "sparkr"

\},

"class": "org.apache.zeppelin.livy.LivySparkSQLInterpreter",

"name": "sql"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2C18DK1TS": \{

"id": "2C18DK1TS",

"name": "python",

"group": "python",

"properties": \{

"zeppelin.python": "python",

"zeppelin.interpreter.localRepo": "/home/haoren/soft/zeppelin/local-repo/2C18DK1TS"

\},

"class": "org.apache.zeppelin.python.PythonInterpreter",

"name": "python"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"2C2CNKWYY": \{

"id": "2C2CNKWYY",

"name": "lens",

"group": "lens",

"properties": \{

"lens.session.cluster.user": "default",

"zeppelin.lens.maxResults": "1000",

"lens.client.dbname": "default",

"lens.query.enable.persistent.resultset": "false",

"zeppelin.lens.maxThreads": "10",

"lens.server.base.url": "http://\\u003chostname\\u003e:\\u003cport\\u003e/lensapi",

"zeppelin.lens.run.concurrent": "true"

\},

"interpreterGroup": \[

"class": "org.apache.zeppelin.lens.LensInterpreter",

"name": "lens"

\],

"dependencies": \[\],

"option": \{

"remote": true,

"perNoteSession": false,

"perNoteProcess": false,

"isExistingProcess": false

\},

"interpreterBindings": \{

"2C13EQNF7": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2A94M5J1Z": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2BQA35CJZ": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C1M3J3XZ": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C2GYXTXY": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C1JVSN53": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C49TEC59": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C1XJZXGT": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C56N6X9C": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C319P4S7": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C38AKEYN": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C53H6D3R": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C5CE6C6M": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C6K6NBSS": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C7DWXVKD": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\],

"2C7YHRF5U": \[

"2C29FZKVG",

"2BYE767ZH",

"2BYECAAP6",

"2BZMQHQTC",

"2BZJZ266V",

"2BY5XFDQC",

"2C18DK1TS",

"2C2CNKWYY",

"2BYKSJZ9C",

"2BY98FHVP",

"2BZ82J2SU",

"2BZ3DPPQX",

"2BZTZF2EV",

"2C1DG6ZVR",

"2C1SV5WUS",

"2BXY3DBVE",

"2C17RTEG8",

"2BZ2H37EF"

\},

"interpreterRepositories": \[

"id": "central",

"type": "default",

"url": "http://repo1.maven.org/maven2/",

"releasePolicy": \{

"enabled": true,

"updatePolicy": "daily",

"checksumPolicy": "warn"

\},

"snapshotPolicy": \{

"enabled": true,

"updatePolicy": "daily",

"checksumPolicy": "warn"

\},

"mirroredRepositories": \[\],

"repositoryManager": false

\},

"id": "local",

"type": "default",

"url": "file:///home/haoren/.m2/repository",

"releasePolicy": \{

"enabled": true,

"updatePolicy": "daily",

"checksumPolicy": "warn"

\},

"snapshotPolicy": \{

"enabled": true,

"updatePolicy": "daily",

"checksumPolicy": "warn"

\},

"mirroredRepositories": \[\],

"repositoryManager": false

大数据分析配置文件

发表评论取消回复

还没有评论，来说两句吧...

相关阅读

相关 Oracle大数据分析性能瓶颈：硬件资源配置不足

相关大数据分析工程师面试 ◀ 数据分析方法

相关大数据分析和数据挖掘区别_大数据分析和数据挖掘之间的区别，大数据的未来范围...

相关 python 读取外部文件行数_[Python] 大文件数据读取分析

相关【数据分析师-数据分析项目案例】大数据预处理

相关大数据分析配置文件

相关大数据分析配置文件

相关房地产大数据分析方法

相关大数据分析技术方案

相关 Kylin大数据分析