diff --git a/README-zh.md b/README-zh.md index 60a37c6f..db50c3f3 100644 --- a/README-zh.md +++ b/README-zh.md @@ -12,7 +12,7 @@ 基于Golang的分布式爬虫管理平台,支持Python、NodeJS、Go、Java、PHP等多种编程语言以及多种爬虫框架。 -[查看演示 Demo](http://crawlab.cn:8080) | [文档](https://tikazyq.github.io/crawlab-docs) +[查看演示 Demo](http://crawlab.cn/demo) | [文档](https://tikazyq.github.io/crawlab-docs) ## 安装 diff --git a/README.md b/README.md index 7ad849e5..3f7fe0b2 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Golang-based distributed web crawler management platform, supporting various languages including Python, NodeJS, Go, Java, PHP and various web crawler frameworks including Scrapy, Puppeteer, Selenium. -[Demo](http://crawlab.cn:8080) | [Documentation](https://tikazyq.github.io/crawlab-docs) +[Demo](http://crawlab.cn/demo) | [Documentation](https://tikazyq.github.io/crawlab-docs) ## Installation diff --git a/backend/main.go b/backend/main.go index b6bb191d..489a17ce 100644 --- a/backend/main.go +++ b/backend/main.go @@ -17,6 +17,7 @@ func main() { // 初始化配置 if err := config.InitConfig(""); err != nil { + log.Error("init config error:" + err.Error()) panic(err) } log.Info("初始化配置成功") @@ -30,6 +31,7 @@ func main() { // 初始化Mongodb数据库 if err := database.InitMongo(); err != nil { + log.Error("init mongodb error:" + err.Error()) debug.PrintStack() panic(err) } @@ -37,6 +39,7 @@ func main() { // 初始化Redis数据库 if err := database.InitRedis(); err != nil { + log.Error("init redis error:" + err.Error()) debug.PrintStack() panic(err) } @@ -45,6 +48,7 @@ func main() { if services.IsMaster() { // 初始化定时任务 if err := services.InitScheduler(); err != nil { + log.Error("init scheduler error:" + err.Error()) debug.PrintStack() panic(err) } @@ -53,6 +57,7 @@ func main() { // 初始化任务执行器 if err := services.InitTaskExecutor(); err != nil { + log.Error("init task executor error:" + err.Error()) debug.PrintStack() panic(err) } @@ -60,12 +65,14 @@ func main() { // 初始化节点服务 if err := services.InitNodeService(); err != nil { + log.Error("init node service error:" + err.Error()) panic(err) } log.Info("初始化节点配置成功") // 初始化爬虫服务 if err := services.InitSpiderService(); err != nil { + log.Error("init spider service error:" + err.Error()) debug.PrintStack() panic(err) } @@ -73,6 +80,7 @@ func main() { // 初始化用户服务 if err := services.InitUserService(); err != nil { + log.Error("init user service error:" + err.Error()) debug.PrintStack() panic(err) } @@ -91,7 +99,7 @@ func main() { app.POST("/nodes/:id", routes.PostNode) // 修改节点 app.GET("/nodes/:id/tasks", routes.GetNodeTaskList) // 节点任务列表 app.GET("/nodes/:id/system", routes.GetSystemInfo) // 节点任务列表 - app.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 + app.DELETE("/nodes/:id", routes.DeleteNode) // 删除节点 // 爬虫 app.GET("/spiders", routes.GetSpiderList) // 爬虫列表 app.GET("/spiders/:id", routes.GetSpider) // 爬虫详情 @@ -138,6 +146,7 @@ func main() { host := viper.GetString("server.host") port := viper.GetString("server.port") if err := app.Run(host + ":" + port); err != nil { + log.Error("run server error:" + err.Error()) panic(err) } } diff --git a/examples/worker/README.md b/examples/worker/README.md index 0a27310c..e8638ccd 100644 --- a/examples/worker/README.md +++ b/examples/worker/README.md @@ -21,3 +21,6 @@ docker build -t crawlab:worker . ``` docker-compose up -d ``` + +如果在多台服务器使用`docker-compose.yml`进行编排,可能出现节点注册不上的问题,因为mac地址冲突了。 +可以使用`networks`定义当前节点的IP段,这样就可以正常注册到redis \ No newline at end of file diff --git a/examples/worker/docker-compose.yml b/examples/worker/docker-compose.yml index 3508b923..63d9c90c 100644 --- a/examples/worker/docker-compose.yml +++ b/examples/worker/docker-compose.yml @@ -5,4 +5,14 @@ services: container_name: crawlab-worker volumes: - $PWD/conf/config.yml:/opt/crawlab/conf/config.yml - - $PWD/crawlab:/usr/local/bin/crawlab \ No newline at end of file + # 二进制包使用源码生成 + - $PWD/crawlab:/usr/local/bin/crawlab + networks: + - crawlabnet + +networks: + crawlabnet: + ipam: + driver: default + config: + - subnet: 172.30.0.0/16 \ No newline at end of file diff --git a/jenkins/docker-compose.yml b/jenkins/docker-compose.yml index fd7d2904..924922a0 100644 --- a/jenkins/docker-compose.yml +++ b/jenkins/docker-compose.yml @@ -25,6 +25,7 @@ services: depends_on: - mongo - redis + replicas: 3 mongo: image: mongo:latest restart: always