webporter 是一个基于垂直爬虫框架 webmagic 的 Java 爬虫应用，旨在提供一套完整的数据爬取，持久化存储和可

[{"title":"（ 61 个子文件 66KB ） webporter 是一个基于垂直爬虫框架 webmagic 的 Java 爬虫应用，旨在提供一套完整的数据爬取，持久化存储和可","children":[{"title":"webporter-master","children":[{"title":"pom.xml <span style='color:#111;'> 12.37KB </span>","children":null,"spread":false},{"title":"webporter-core","children":[{"title":"pom.xml <span style='color:#111;'> 939B </span>","children":null,"spread":false},{"title":"src","children":[{"title":"test","children":[{"title":"resources","children":[{"title":"line-file.html <span style='color:#111;'> 11B </span>","children":null,"spread":false},{"title":"log4j.xml <span style='color:#111;'> 669B </span>","children":null,"spread":false},{"title":"config.json <span style='color:#111;'> 141B </span>","children":null,"spread":false},{"title":"basic-config.json <span style='color:#111;'> 357B </span>","children":null,"spread":false},{"title":"2daa3de3d1a21aed6df5ee2f629b314a.html <span style='color:#111;'> 11.49KB </span>","children":null,"spread":false}],"spread":true},{"title":"java","children":[{"title":"com","children":[{"title":"brianway","children":[{"title":"webporter","children":[{"title":"configure","children":[{"title":"SiteConfigurationTest.java <span style='color:#111;'> 504B </span>","children":null,"spread":false},{"title":"BasicConfigurationTest.java <span style='color:#111;'> 802B </span>","children":null,"spread":false}],"spread":true},{"title":"data","children":[{"title":"DemoDataProcessorTest.java <span style='color:#111;'> 726B </span>","children":null,"spread":false},{"title":"FileRawInputTest.java <span style='color:#111;'> 396B </span>","children":null,"spread":false}],"spread":true},{"title":"BaseTest.java <span style='color:#111;'> 245B </span>","children":null,"spread":false},{"title":"util","children":[{"title":"FileHelperTest.java <span style='color:#111;'> 2.89KB </span>","children":null,"spread":false},{"title":"StringHelperTest.java <span style='color:#111;'> 404B </span>","children":null,"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"main","children":[{"title":"resources","children":[{"title":"log4j.xml <span style='color:#111;'> 669B </span>","children":null,"spread":false},{"title":"config.json <span style='color:#111;'> 161B </span>","children":null,"spread":false}],"spread":true},{"title":"java","children":[{"title":"com","children":[{"title":"brianway","children":[{"title":"webporter","children":[{"title":"configure","children":[{"title":"AbstractConfiguration.java <span style='color:#111;'> 680B </span>","children":null,"spread":false},{"title":"BasicConfiguration.java <span style='color:#111;'> 877B </span>","children":null,"spread":false},{"title":"SiteConfiguration.java <span style='color:#111;'> 669B </span>","children":null,"spread":false}],"spread":true},{"title":"data","children":[{"title":"ConsoleOutpipeline.java <span style='color:#111;'> 394B </span>","children":null,"spread":false},{"title":"DataProcessor.java <span style='color:#111;'> 653B </span>","children":null,"spread":false},{"title":"RawInput.java <span style='color:#111;'> 214B </span>","children":null,"spread":false},{"title":"BaseAssembler.java <span style='color:#111;'> 5.51KB </span>","children":null,"spread":false},{"title":"DemoDataProcessor.java <span style='color:#111;'> 1.01KB </span>","children":null,"spread":false},{"title":"DataFlow.java <span style='color:#111;'> 486B </span>","children":null,"spread":false},{"title":"DuplicateRemover.java <span style='color:#111;'> 173B </span>","children":null,"spread":false},{"title":"FileRawInput.java <span style='color:#111;'> 563B </span>","children":null,"spread":false},{"title":"OutPipeline.java <span style='color:#111;'> 220B </span>","children":null,"spread":false},{"title":"HashSetDuplicateRemover.java <span style='color:#111;'> 634B </span>","children":null,"spread":false}],"spread":true},{"title":"util","children":[{"title":"FileHelper.java <span style='color:#111;'> 2.29KB </span>","children":null,"spread":false},{"title":"StringHelper.java <span style='color:#111;'> 602B </span>","children":null,"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"webporter-data-elasticsearch","children":[{"title":"pom.xml <span style='color:#111;'> 1.11KB </span>","children":null,"spread":false},{"title":"src","children":[{"title":"main","children":[{"title":"resources","children":[{"title":"log4j.xml <span style='color:#111;'> 669B </span>","children":null,"spread":false}],"spread":true},{"title":"java","children":[{"title":"com","children":[{"title":"brianway","children":[{"title":"webporter","children":[{"title":"data","children":[{"title":"elasticsearch","children":[{"title":"Document.java <span style='color:#111;'> 560B </span>","children":null,"spread":false},{"title":"ElasticsearchUploader.java <span style='color:#111;'> 3.42KB </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"webporter-collector-zhihu","children":[{"title":"pom.xml <span style='color:#111;'> 1.33KB </span>","children":null,"spread":false},{"title":"src","children":[{"title":"test","children":[{"title":"resources","children":[{"title":"log4j.xml <span style='color:#111;'> 669B </span>","children":null,"spread":false},{"title":"followee-folder","children":[{"title":"2daa3de3d1a21aed6df5ee2f629b314a.html <span style='color:#111;'> 11.49KB </span>","children":null,"spread":false}],"spread":true},{"title":"member.json <span style='color:#111;'> 3.80KB </span>","children":null,"spread":false},{"title":"config.json <span style='color:#111;'> 357B </span>","children":null,"spread":false},{"title":"followee.json <span style='color:#111;'> 17.34KB </span>","children":null,"spread":false},{"title":"another-config.json <span style='color:#111;'> 357B </span>","children":null,"spread":false}],"spread":true},{"title":"java","children":[{"title":"com","children":[{"title":"brianway","children":[{"title":"webporter","children":[{"title":"collector","children":[{"title":"MemberURLTokenGeneratorTest.java <span style='color:#111;'> 790B </span>","children":null,"spread":false},{"title":"ZhihuMemberDataProcessorTest.java <span style='color:#111;'> 933B </span>","children":null,"spread":false},{"title":"ZhihuFolloweePageProcessorTest.java <span style='color:#111;'> 2.99KB </span>","children":null,"spread":false},{"title":"BaseTest.java <span style='color:#111;'> 260B </span>","children":null,"spread":false},{"title":"ZhihuConfigurationTest.java <span style='color:#111;'> 1.49KB </span>","children":null,"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":"main","children":[{"title":"resources","children":[{"title":"log4j.xml <span style='color:#111;'> 669B </span>","children":null,"spread":false},{"title":"config.json <span style='color:#111;'> 363B </span>","children":null,"spread":false}],"spread":true},{"title":"java","children":[{"title":"com","children":[{"title":"brianway","children":[{"title":"webporter","children":[{"title":"collector","children":[{"title":"zhihu","children":[{"title":"processor","children":[{"title":"ZhihuMemberDataProcessor.java <span style='color:#111;'> 2.36KB </span>","children":null,"spread":false},{"title":"MemberURLTokenGenerator.java <span style='color:#111;'> 3.68KB </span>","children":null,"spread":false},{"title":"ZhihuFolloweeDataProcessor.java <span style='color:#111;'> 1.33KB </span>","children":null,"spread":false}],"spread":false},{"title":"ZhihuConfiguration.java <span style='color:#111;'> 1.07KB </span>","children":null,"spread":false},{"title":"upload","children":[{"title":"FolloweeUploader.java <span style='color:#111;'> 1.33KB </span>","children":null,"spread":false},{"title":"MemberUploader.java <span style='color:#111;'> 1.37KB </span>","children":null,"spread":false},{"title":"ZhihuElasticsearchUploader.java <span style='color:#111;'> 1.27KB </span>","children":null,"spread":false}],"spread":false},{"title":"download","children":[{"title":"ZhihuFolloweePageProcessor.java <span style='color:#111;'> 2.77KB </span>","children":null,"spread":false},{"title":"ZhihuPipeline.java <span style='color:#111;'> 1.58KB </span>","children":null,"spread":false},{"title":"ZhihuMemberPageProcessor.java <span style='color:#111;'> 2.85KB </span>","children":null,"spread":false}],"spread":false}],"spread":false}],"spread":false}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true}],"spread":true},{"title":".gitignore <span style='color:#111;'> 141B </span>","children":null,"spread":false},{"title":"README.md <span style='color:#111;'> 5.37KB </span>","children":null,"spread":false}],"spread":true}],"spread":true}]