基于storm实时热点统计的分布式并行缓存预热
一、基于nginx+lua完成商品详情页访问流量实时上报kafka的开发
====================================
在nginx这一层，接收到访问请求的时候，就把请求的流量上报发送给kafka
这样的话，storm才能去消费kafka中的实时的访问日志，然后去进行缓存热数据的统计
用得技术方案非常简单，从lua脚本直接创建一个kafka producer，发送数据到kafka
```
wget https://github.com/doujiang24/lua-resty-kafka/archive/master.zip
yum install -y unzip
unzip lua-resty-kafka-master.zip
cp -rf /usr/local/lua-resty-kafka-master/lib/resty /usr/hello/lualib
nginx -s reload
local cjson = require("cjson")
local producer = require("resty.kafka.producer")
local broker_list = {
{ host = "192.168.31.187", port = 9092 },
{ host = "192.168.31.19", port = 9092 },
{ host = "192.168.31.227", port = 9092 }
}
local log_json = {}
log_json["headers"] = ngx.req.get_headers()
log_json["uri_args"] = ngx.req.get_uri_args()
log_json["body"] = ngx.req.read_body()
log_json["http_version"] = ngx.req.http_version()
log_json["method"] =ngx.req.get_method()
log_json["raw_reader"] = ngx.req.raw_header()
log_json["body_data"] = ngx.req.get_body_data()
local message = cjson.encode(log_json);
local productId = ngx.req.get_uri_args()["productId"]
local async_producer = producer:new(broker_list, { producer_type = "async" })
local ok, err = async_producer:send("access-log", productId, message)
if not ok then
ngx.log(ngx.ERR, "kafka send err:", err)
return
end
```
两台机器上都这样做，才能统一上报流量到kafka
```
bin/kafka-topics.sh --zookeeper 192.168.31.187:2181,192.168.31.19:2181,192.168.31.227:2181 --topic access-log --replication-factor 1 --partitions 1 --create
bin/kafka-console-consumer.sh --zookeeper 192.168.31.187:2181,192.168.31.19:2181,192.168.31.227:2181 --topic access-log --from-beginning
```
（1）kafka在187上的节点死掉了，可能是虚拟机的问题，杀掉进程，重新启动一下
nohup bin/kafka-server-start.sh config/server.properties &
（2）需要在nginx.conf中，http部分，加入resolver 8.8.8.8;
（3）需要在kafka中加入advertised.host.name = 192.168.31.187，重启三个kafka进程
（4）需要启动eshop-cache缓存服务，因为nginx中的本地缓存可能不在了
二、基于storm+kafka完成商品访问次数实时统计拓扑的开发
==============
文件下载
资源详情
[{"title":"（ 80 个子文件 30.71MB ） 基于storm实时热点统计的分布式并行缓存预热","children":[{"title":"代码","children":[{"title":"eshop-cache","children":[{"title":"pom.xml <span style='color:#111;'> 3.71KB </span>","children":null,"spread":false},{"title":"target","children":[{"title":"test-classes","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"cache","children":[{"title":"ZooKeeperTest.class <span style='color:#111;'> 626B </span>","children":null,"spread":false},{"title":"JedisTest.class <span style='color:#111;'> 1.26KB </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"classes","children":[{"title":"templates","children":[{"title":"hello.html <span style='color:#111;'> 0B </span>","children":null,"spread":false}],"spread":true},{"title":"Application.properties <span style='color:#111;'> 186B </span>","children":null,"spread":false},{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"cache","children":[{"title":"model","children":[{"title":"ShopInfo.class <span style='color:#111;'> 1.68KB </span>","children":null,"spread":false},{"title":"ProductInfo.class <span style='color:#111;'> 2.93KB </span>","children":null,"spread":false}],"spread":true},{"title":"zk","children":[{"title":"ZooKeeperSession.class <span style='color:#111;'> 5.39KB </span>","children":null,"spread":false},{"title":"ZooKeeperSession$ZooKeeperWatcher.class <span style='color:#111;'> 1.75KB </span>","children":null,"spread":false},{"title":"ZooKeeperSession$Singleton.class <span style='color:#111;'> 702B </span>","children":null,"spread":false}],"spread":true},{"title":"rebuild","children":[{"title":"RebuildCacheQueue.class <span style='color:#111;'> 1.52KB </span>","children":null,"spread":false},{"title":"RebuildCacheThread.class <span style='color:#111;'> 3.08KB </span>","children":null,"spread":false},{"title":"RebuildCacheQueue$Singleton.class <span style='color:#111;'> 733B </span>","children":null,"spread":false}],"spread":false},{"title":"prewarm","children":[{"title":"CachePrewarmThread.class <span style='color:#111;'> 4.22KB </span>","children":null,"spread":false}],"spread":false},{"title":"kafka","children":[{"title":"KafkaConsumer.class <span style='color:#111;'> 2.62KB </span>","children":null,"spread":false},{"title":"KafkaMessageProcessor.class <span style='color:#111;'> 5.70KB </span>","children":null,"spread":false}],"spread":false},{"title":"controller","children":[{"title":"CacheController.class <span style='color:#111;'> 3.75KB </span>","children":null,"spread":false}],"spread":false},{"title":"configuration","children":[{"title":"CacheConfiguration.class <span style='color:#111;'> 1.47KB </span>","children":null,"spread":false}],"spread":false},{"title":"service","children":[{"title":"CacheService.class <span style='color:#111;'> 865B </span>","children":null,"spread":false},{"title":"impl","children":[{"title":"CacheServiceImpl.class <span style='color:#111;'> 3.55KB </span>","children":null,"spread":false}],"spread":false}],"spread":false},{"title":"Application.class <span style='color:#111;'> 3.44KB </span>","children":null,"spread":false},{"title":"spring","children":[{"title":"SpringContext.class <span style='color:#111;'> 684B </span>","children":null,"spread":false}],"spread":false},{"title":"listener","children":[{"title":"InitListener.class <span style='color:#111;'> 1.53KB </span>","children":null,"spread":false}],"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true},{"title":"ehcache.xml <span style='color:#111;'> 2.81KB </span>","children":null,"spread":false},{"title":"mybatis","children":[{"title":"UserMapper.xml <span style='color:#111;'> 230B </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true},{"title":".settings","children":[{"title":"org.eclipse.m2e.core.prefs <span style='color:#111;'> 90B </span>","children":null,"spread":false},{"title":"org.eclipse.core.resources.prefs <span style='color:#111;'> 191B </span>","children":null,"spread":false},{"title":"org.eclipse.jdt.core.prefs <span style='color:#111;'> 243B </span>","children":null,"spread":false}],"spread":true},{"title":"src","children":[{"title":"test","children":[{"title":"resources","children":null,"spread":false},{"title":"java","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"cache","children":[{"title":"JedisTest.java <span style='color:#111;'> 761B </span>","children":null,"spread":false},{"title":"ZooKeeperTest.java <span style='color:#111;'> 239B </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"main","children":[{"title":"resources","children":[{"title":"templates","children":[{"title":"hello.html <span style='color:#111;'> 0B </span>","children":null,"spread":false}],"spread":true},{"title":"Application.properties <span style='color:#111;'> 186B </span>","children":null,"spread":false},{"title":"ehcache.xml <span style='color:#111;'> 2.81KB </span>","children":null,"spread":false},{"title":"mybatis","children":[{"title":"UserMapper.xml <span style='color:#111;'> 230B </span>","children":null,"spread":false}],"spread":true}],"spread":true},{"title":"java","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"cache","children":[{"title":"model","children":[{"title":"ProductInfo.java <span style='color:#111;'> 2.01KB </span>","children":null,"spread":false},{"title":"ShopInfo.java <span style='color:#111;'> 917B </span>","children":null,"spread":false}],"spread":false},{"title":"zk","children":[{"title":"ZooKeeperSession.java <span style='color:#111;'> 6.08KB </span>","children":null,"spread":false}],"spread":false},{"title":"rebuild","children":[{"title":"RebuildCacheQueue.java <span style='color:#111;'> 1.09KB </span>","children":null,"spread":false},{"title":"RebuildCacheThread.java <span style='color:#111;'> 2.00KB </span>","children":null,"spread":false}],"spread":false},{"title":"prewarm","children":[{"title":"CachePrewarmThread.java <span style='color:#111;'> 3.02KB </span>","children":null,"spread":false}],"spread":false},{"title":"Application.java <span style='color:#111;'> 2.99KB </span>","children":null,"spread":false},{"title":"kafka","children":[{"title":"KafkaConsumer.java <span style='color:#111;'> 1.69KB </span>","children":null,"spread":false},{"title":"KafkaMessageProcessor.java <span style='color:#111;'> 5.97KB </span>","children":null,"spread":false}],"spread":false},{"title":"controller","children":[{"title":"CacheController.java <span style='color:#111;'> 3.09KB </span>","children":null,"spread":false}],"spread":false},{"title":"configuration","children":[{"title":"CacheConfiguration.java <span style='color:#111;'> 1.06KB </span>","children":null,"spread":false}],"spread":false},{"title":"service","children":[{"title":"CacheService.java <span style='color:#111;'> 1.70KB </span>","children":null,"spread":false},{"title":"impl","children":[{"title":"CacheServiceImpl.java <span style='color:#111;'> 3.33KB </span>","children":null,"spread":false}],"spread":false}],"spread":false},{"title":"spring","children":[{"title":"SpringContext.java <span style='color:#111;'> 499B </span>","children":null,"spread":false}],"spread":false},{"title":"listener","children":[{"title":"InitListener.java <span style='color:#111;'> 1.11KB </span>","children":null,"spread":false}],"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":".project <span style='color:#111;'> 563B </span>","children":null,"spread":false},{"title":".classpath <span style='color:#111;'> 1.40KB </span>","children":null,"spread":false}],"spread":true},{"title":"eshop-storm","children":[{"title":"pom.xml <span style='color:#111;'> 4.56KB </span>","children":null,"spread":false},{"title":"target","children":[{"title":"test-classes","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"eshop_storm","children":[{"title":"AppTest.class <span style='color:#111;'> 643B </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"eshop-storm-0.0.1-SNAPSHOT.jar <span style='color:#111;'> 16.92MB </span>","children":null,"spread":false},{"title":"surefire","children":null,"spread":false},{"title":"eshop-storm-0.0.1-SNAPSHOT-shaded.jar <span style='color:#111;'> 16.92MB </span>","children":null,"spread":false},{"title":"classes","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"storm","children":[{"title":"zk","children":[{"title":"ZooKeeperSession.class <span style='color:#111;'> 4.13KB </span>","children":null,"spread":false},{"title":"ZooKeeperSession$ZooKeeperWatcher.class <span style='color:#111;'> 1.73KB </span>","children":null,"spread":false},{"title":"ZooKeeperSession$Singleton.class <span style='color:#111;'> 702B </span>","children":null,"spread":false}],"spread":false},{"title":"HotProductTopology.class <span style='color:#111;'> 2.43KB </span>","children":null,"spread":false},{"title":"spout","children":[{"title":"AccessLogKafkaSpout.class <span style='color:#111;'> 5.01KB </span>","children":null,"spread":false},{"title":"AccessLogKafkaSpout$KafkaMessageProcessor.class <span style='color:#111;'> 2.06KB </span>","children":null,"spread":false}],"spread":false},{"title":"bolt","children":[{"title":"LogParseBolt.class <span style='color:#111;'> 2.69KB </span>","children":null,"spread":false},{"title":"ProductCountBolt$ProductCountThread.class <span style='color:#111;'> 3.91KB </span>","children":null,"spread":false},{"title":"ProductCountBolt.class <span style='color:#111;'> 4.40KB </span>","children":null,"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"maven-archiver","children":[{"title":"pom.properties <span style='color:#111;'> 126B </span>","children":null,"spread":false}],"spread":true},{"title":"surefire-reports","children":[{"title":"TEST-com.roncoo.eshop.eshop_storm.AppTest.xml <span style='color:#111;'> 4.69KB </span>","children":null,"spread":false},{"title":"com.roncoo.eshop.eshop_storm.AppTest.txt <span style='color:#111;'> 284B </span>","children":null,"spread":false}],"spread":true},{"title":"original-eshop-storm-0.0.1-SNAPSHOT.jar <span style='color:#111;'> 17.44KB </span>","children":null,"spread":false}],"spread":true},{"title":".settings","children":[{"title":"org.eclipse.m2e.core.prefs <span style='color:#111;'> 90B </span>","children":null,"spread":false},{"title":"org.eclipse.core.resources.prefs <span style='color:#111;'> 119B </span>","children":null,"spread":false},{"title":"org.eclipse.jdt.core.prefs <span style='color:#111;'> 243B </span>","children":null,"spread":false}],"spread":true},{"title":"src","children":[{"title":"test","children":[{"title":"java","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"eshop_storm","children":[{"title":"AppTest.java <span style='color:#111;'> 694B </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"main","children":[{"title":"java","children":[{"title":"com","children":[{"title":"roncoo","children":[{"title":"eshop","children":[{"title":"storm","children":[{"title":"zk","children":[{"title":"ZooKeeperSession.java <span style='color:#111;'> 4.38KB </span>","children":null,"spread":false}],"spread":false},{"title":"spout","children":[{"title":"AccessLogKafkaSpout.java <span style='color:#111;'> 3.62KB </span>","children":null,"spread":false}],"spread":false},{"title":"HotProductTopology.java <span style='color:#111;'> 1.43KB </span>","children":null,"spread":false},{"title":"bolt","children":[{"title":"LogParseBolt.java <span style='color:#111;'> 1.59KB </span>","children":null,"spread":false},{"title":"ProductCountBolt.java <span style='color:#111;'> 5.57KB </span>","children":null,"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":".project <span style='color:#111;'> 563B </span>","children":null,"spread":false},{"title":".classpath <span style='color:#111;'> 1023B </span>","children":null,"spread":false},{"title":"dependency-reduced-pom.xml <span style='color:#111;'> 4.31KB </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}]
评论信息
其他资源
- 现代空间谱估计算法的应用matlab程序仿真
- 激光打标软件Ezcad2.7.6软件 破解版,免狗
- 从三维数组中提取出任意二维的数据，并保存在新的二维矩阵中
- 超赞的200个经典C#WinForm实例源码
- 基于labview的AD转换
- 网络规划设计师历年真题+解析+50篇论文范文.rar
- 2018年全国367个城市每日每时空气质量相关数据
- duilib 做的Demo ，支持进度条，以及动态修改xml 的内容
- 矩阵计算器课程设计（C++）
- 小波分析算法与应用-程正兴-西安交通大学出版
- opencv_haartraining等可执行文件
- USB通讯CCID协议
- 从注意力维度谈动效设计
- AnyTXT.Searcher.1.2.394.exe
- 流程节点图高亮.zip
- SSI 增删改查 范例
- libaio-0.3.104.tar.gz
- 黑苹果安装工具集合包
- 定时自动压缩备份文件并发送电子邮件工具V1.0
- VC++的.cpp文件调用CUDA的.cu文件中的函数
- 北邮数据库上机实验
- Database System Implementation(数据库系统实现算法)
- select2控件+拼音智能检索示例
- servlet&jsp学习笔记项目源码
免责申明
【只为小站】的资源来自网友分享，仅供学习研究，请务必在下载后24小时内给予删除，不得用于其他任何用途，否则后果自负。基于互联网的特殊性，【只为小站】 无法对用户传输的作品、信息、内容的权属或合法性、合规性、真实性、科学性、完整权、有效性等进行实质审查；无论 【只为小站】 经营者是否已进行审查，用户均应自行承担因其传输的作品、信息、内容而可能或已经产生的侵权或权属纠纷等法律责任。
本站所有资源不代表本站的观点或立场，基于网友分享，根据中国法律《信息网络传播权保护条例》第二十二条之规定，若资源存在侵权或相关问题请联系本站客服人员，zhiweidada#qq.com，请把#换成@，本站将给予最大的支持与配合，做到及时反馈和处理。关于更多版权及免责申明参见 版权及免责申明
本站所有资源不代表本站的观点或立场，基于网友分享，根据中国法律《信息网络传播权保护条例》第二十二条之规定，若资源存在侵权或相关问题请联系本站客服人员，zhiweidada#qq.com，请把#换成@，本站将给予最大的支持与配合，做到及时反馈和处理。关于更多版权及免责申明参见 版权及免责申明