爬虫 spider11——搭建分布式架构通过feign技术,开发服务消费者
搭建分布式架構,把3中開發的服務提供者,注冊到eureka server(三臺,7001,7002,7003)
開發服務消費者(可以直接訪問3中的服務),調試成功后,
通過feign技術,開發服務消費者,并注冊到eureka server中。
?
接https://blog.csdn.net/qq_41946557/article/details/102584047
搭建elk平臺,開發服務提供者
和https://blog.csdn.net/qq_41946557/article/details/102573282
爬取指定數據,去重復,并存儲到mysql
?
- 搭建分布式架構,把3中開發的服務提供者,注冊到eureka server(三臺,7001,7002,7003)
1.三臺eureka server
Pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
?????????xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 ????<parent>
 ????????<artifactId>com.henu.spider</artifactId>
 ????????<groupId>com.henu</groupId>
 ????????<version>1.0-SNAPSHOT</version>
 ????</parent>
 ????<modelVersion>4.0.0</modelVersion>
 ????<artifactId>spider_eureka_7001</artifactId>
 ????<properties>
 ????????<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 ????????<maven.compiler.source>1.8</maven.compiler.source>
 ????????<maven.compiler.target>1.8</maven.compiler.target>
 ????</properties>
 ????<dependencies>
 ????????<dependency>
 ????????????<groupId>org.springframework.boot</groupId>
 ????????????<artifactId>spring-boot-starter-web</artifactId>
 ????????</dependency>
 ????????<!--eureka-server服務端 -->
????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????????<artifactId>spring-cloud-starter-eureka-server</artifactId>
 ????????</dependency>
 ????????<!-- 修改后立即生效,熱部署 -->
????????<dependency>
 ????????????<groupId>org.springframework</groupId>
 ????????????<artifactId>springloaded</artifactId>
 ????????</dependency>
 ????????<dependency>
 ????????????<groupId>org.springframework.boot</groupId>
 ????????????<artifactId>spring-boot-devtools</artifactId>
 ????????</dependency>
 ????</dependencies>
</project>
?
EurekaServer7001App
package com.henu.springcloud;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.netflix.eureka.server.EnableEurekaServer;
@SpringBootApplication
@EnableEurekaServer//EurekaServer服務器端啟動類,接受其它微服務注冊進來
public class EurekaServer7001App {
 ????public static void main(String[] args) {
????????SpringApplication.run(EurekaServer7001App.class, args);
????}
}
?
application.yml
server:
 ??port: 7001
eureka:
 ??instance:
 ????hostname: eureka7001.com #eureka服務端的實例名稱
??client:
 ????register-with-eureka: false #false表示不向注冊中心注冊自己。
????fetch-registry: false #false表示自己端就是注冊中心,我的職責就是維護服務實例,并不需要去檢索服務
????service-url:
 ??????#defaultZone: http://${eureka.instance.hostname}:${server.port}/eureka/ ???????#設置與Eureka Server交互的地址查詢服務和注冊服務都需要依賴這個地址。
??????defaultZone: http://eureka7002.com:7002/eureka/,http://eureka7003.com:7003/eureka/
?
spider_eureka_7002和spider_eureka_7003中
修改每個配置,如server.port、eureka.instance.hostname、eureka.client.service-url.defaultZone
?
?
2.修改spider_provider
添加pom.xml
<!-- 將微服務provider側注冊進eureka -->
<dependency>
 ????<groupId>org.springframework.cloud</groupId>
 ????<artifactId>spring-cloud-starter-eureka</artifactId>
</dependency>
<dependency>
 ????<groupId>org.springframework.cloud</groupId>
 ????<artifactId>spring-cloud-starter-config</artifactId>
</dependency>
<!--****************************************-->
<!-- ?hystrix -->
<dependency>
 ????<groupId>org.springframework.cloud</groupId>
 ????<artifactId>spring-cloud-starter-hystrix</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>springloaded</artifactId>
</dependency>
<dependency>
 ????<groupId>org.springframework.boot</groupId>
 ????<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
?
添加application.yml
eureka:
 ??client: #客戶端注冊進eureka服務列表內
????service-url:
 ??????#defaultZone: http://localhost:7001/eureka
??????defaultZone: http://localhost:7001/eureka,http://localhost:7002/eureka,http://localhost:7003/eureka
 ??instance:
 ????instance-id: spider_provider
 ????prefer-ip-address: true ????#訪問路徑可以顯示IP地址
?
修改SpringbootElasticsearchApplication啟動類
package com.henu.es;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.client.circuitbreaker.EnableCircuitBreaker;
import org.springframework.cloud.netflix.eureka.EnableEurekaClient;
/**
?/**
?* 操作elasticsearch有兩種方式:
?* (1)jest:默認不生效,需要導入包io.searchbox.jest
?* ???配置application.properties,測試添加文檔和查詢文檔
?* (2)spring-data-es:導入spring-data-elasticsearch包
?* ???配置application.properties:cluster-name ?cluster-nodes
?* ???啟動要是報錯:可能是版本不匹配
?* ???兩種用法:
?* ???(1)編寫接口繼承elasticsearchRepository
?* ????(2) elasticsearchTemplate
?* (3)spring-data-es CRUD + 分頁 + 高亮的練習
?*
?*/
@SpringBootApplication
@EnableEurekaClient //本服務啟動后會自動注冊進eureka服務中
@EnableCircuitBreaker //啟用hystrix熔斷機制
public class SpringbootElasticsearchApplication {
 ????public static void main(String[] args) {
 ????????// 避免netty沖突
????????System.setProperty("es.set.netty.runtime.available.processors", "false");
????????SpringApplication.run(SpringbootElasticsearchApplication.class, args);
????}
}
?
修改SpiderController
@RestController
public class SpiderController {
 ????@RequestMapping("/search")
 ????@HystrixCommand(fallbackMethod = "processHystrixDeptGet")//服務熔斷
????public String search(@RequestParam(value = "keyword")String keyword, @RequestParam(value="currentPage",defaultValue = "1") int currentPage, @RequestParam(value="pageSize",defaultValue = "10") int pageSize){
????????System.out.println("好好學習,天天向上:"+keyword);
????????QueryBuilder queryBuilder = QueryBuilders.matchQuery("intro", keyword);
????????EsPage esPage = ElasticsearchUtil.searchDataPage("spider", currentPage, pageSize, queryBuilder, "id,appid,title,intro,url,source,updatetime", "id", "intro");
 ????????return esPage.toString();
????}
 ????public String processHystrixDeptGet(@RequestParam(value = "keyword")String keyword, @RequestParam(value="currentPage",defaultValue = "1") int currentPage, @RequestParam(value="pageSize",defaultValue = "10") int pageSize){
????????System.out.println("error");
 ????????return "error111";
????}
}
?
- 開發服務消費者(可以直接訪問3中的服務),調試成功后,通過feign技術,開發服務消費者,并注冊到eureka server中。
(1)配置類:
RandomRule_Five
public class RandomRule_Five extends AbstractLoadBalancerRule {
 ????private int total = 0; ???//總共被調用的次數,目前要求每臺被調用5次
????private int currentIndex = 0;//當前提供服務的機器號
????public Server choose(ILoadBalancer lb, Object key) {
 ????????if (lb == null) {
 ????????????return null;
????????}
????????Server server = null;
 ????????while (server == null) {
 ????????????if (Thread.interrupted()) {
 ????????????????return null;
????????????}
????????????List<Server> upList = lb.getReachableServers();
????????????List<Server> allList = lb.getAllServers();
 ????????????int serverCount = allList.size();
 ????????????if (serverCount == 0) {
 ????????????????/*
?????????????????* No servers. End regardless of pass, because subsequent passes
?????????????????* only get more restrictive.
?????????????????*/
????????????????return null;
????????????}
// ???????????int index = rand.nextInt(serverCount);
// ???????????server = upList.get(index);
????????????if(total < 5)
????????????{
????????????????server = upList.get(currentIndex);
 ????????????????total++;
????????????}else {
 ????????????????total = 0;
 ????????????????currentIndex++;
 ????????????????if(currentIndex >= upList.size())
????????????????{
 ????????????????????currentIndex = 0;
????????????????}
????????????}
 ????????????if (server == null) {
 ????????????????/*
?????????????????* The only time this should happen is if the server list were
?????????????????* somehow trimmed. This is a transient condition. Retry after
?????????????????* yielding.
?????????????????*/
????????????????Thread.yield();
 ????????????????continue;
????????????}
 ????????????if (server.isAlive()) {
 ????????????????return (server);
????????????}
 ????????????// Shouldn't actually happen.. but must be transient or a bug.
????????????server = null;
????????????Thread.yield();
????????}
 ????????return server;
????}
 ????@Override
 ????public Server choose(Object key) {
 ????????return choose(getLoadBalancer(), key);
????}
 ????@Override
 ????public void initWithNiwsConfig(IClientConfig clientConfig) {
????}
}
?
RibbonConfig
@SpringBootConfiguration
public class RibbonConfig {
 ????@Bean
 ????public IRule getRule(){
 ????????//return ?new RandomRule();指定負載均衡算法
????????return ?new RandomRule_Five();
????}
}
?
(2)控制層
NewsControllerConsumer
@RestController
public class NewsControllerConsumer {
 ????@Autowired
 ????private SpiderClientService service;
 ????/**
?????* 在客戶端執行遠程訪問,訪問服務發現接口
?????*
?????* @return
?????*/
????@RequestMapping(value = "/consumer/search")
 ????public String list(@RequestParam(value = "keyword") String keyword, @RequestParam(value = "currentPage", defaultValue = "1") int currentPage, @RequestParam(value = "pageSize", defaultValue = "10") int pageSize) {
????????System.out.println("nihao");
 ????????//請求轉發
????????return service.search(keyword, currentPage, pageSize);
????}
}
?
(3)工廠類
SpiderClientServiceFallbackFactory
@Component
public class SpiderClientServiceFallbackFactory implements FallbackFactory<SpiderClientService> {
 ????@Override
 ????public SpiderClientService create(Throwable throwable) {
 ????????return new SpiderClientService(){
 ????????????@Override
 ????????????public String search(String keyword, int currentPage, int pageSize) {
 ????????????????return "dddddddd";
????????????}
????????};
????}
}
?
SpiderClientService
//使用feign,并制定服務
@FeignClient(value = "SPIDER8001",fallbackFactory = SpiderClientServiceFallbackFactory.class)
public interface SpiderClientService {
 ????@RequestMapping("/search")
????String search(@RequestParam(value = "keyword")String keyword, @RequestParam(value="currentPage",defaultValue = "1") int currentPage, @RequestParam(value="pageSize",defaultValue = "10") int pageSize);
}
?
(5)啟動類
SpiderConsumer9001App
@SpringBootApplication
@EnableEurekaClient
@RibbonClient(name="SPIDER8001")
@EnableFeignClients(basePackageClasses = SpiderClientService.class)
@EnableHystrixDashboard
public class SpiderConsumer9001App {
 ????public static void main(String[] args)
????{
????????SpringApplication.run(SpiderConsumer9001App.class, args);
????}
}
?
server:
 ??port: 9001
 ??context-path: /
eureka:
 ??client: #客戶端注冊進eureka服務列表內
????service-url:
 ??????#defaultZone: http://localhost:7001/eureka
??????defaultZone: http://localhost:7001/eureka,http://localhost:7002/eureka,http://localhost:7003/eureka
 ????register-with-eureka: false
feign:
 ??hystrix:
 ????enabled: true #啟用hystrix
?
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
?????????xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 ????<parent>
 ????????<artifactId>com.henu.spider</artifactId>
 ????????<groupId>com.henu</groupId>
 ????????<version>1.0-SNAPSHOT</version>
 ????</parent>
 ????<modelVersion>4.0.0</modelVersion>
 ????<artifactId>spider_consumer_01</artifactId>
 ????<packaging>war</packaging>
 ????<name>spider_consumer_01 Maven Webapp</name>
 ????<!-- FIXME change it to the project's website -->
????<url>http://www.example.com</url>
 ????<properties>
 ????????<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 ????????<maven.compiler.source>1.8</maven.compiler.source>
 ????????<maven.compiler.target>1.8</maven.compiler.target>
 ????</properties>
 ????<dependencies>
 ????????<dependency>
 ????????????<groupId>org.springframework.boot</groupId>
 ????????????<artifactId>spring-boot-starter-web</artifactId>
 ????????</dependency>
 ????????<!--*************************************************************-->
????????<!-- 將微服務provider側注冊進eureka -->
????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????????<artifactId>spring-cloud-starter-eureka</artifactId>
 ????????</dependency>
 ????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????????<artifactId>spring-cloud-starter-config</artifactId>
 ????????</dependency>
 ????????<!--************************************-->
????????<!-- Ribbon相關 -->
????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????????<artifactId>spring-cloud-starter-ribbon</artifactId>
 ????????</dependency>
 ????????<!-- feign相關 -->
????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????????<artifactId>spring-cloud-starter-feign</artifactId>
 ????????</dependency>
 ????????<!-- hystrix和?hystrix-dashboard相關-->
????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????????<artifactId>spring-cloud-starter-hystrix</artifactId>
 ????????</dependency>
 ????????<dependency>
 ????????????<groupId>org.springframework.cloud</groupId>
 ????????<artifactId>spring-cloud-starter-hystrix-dashboard</artifactId>
 ????????</dependency>
 ????</dependencies>
 ????<build>
 ????????<plugins>
 ????????????<plugin>
 ????????????????<groupId>org.springframework.boot</groupId>
 ????????????????<artifactId>spring-boot-maven-plugin</artifactId>
 ????????????</plugin>
 ????????</plugins>
 ????</build>
</project>
?
全部啟動成功后!
展示結果:
首先eureka
?訪問http://localhost:9001/consumer/search?keyword=黃
ok!!!繼續加油。
?
總結
以上是生活随笔為你收集整理的爬虫 spider11——搭建分布式架构通过feign技术,开发服务消费者的全部內容,希望文章能夠幫你解決所遇到的問題。
 
                            
                        - 上一篇: 爬虫 spider10——搭建elk平台
- 下一篇: Redis布隆过滤器
