加入挑战

This commit is contained in:
marvzhang
2020-03-19 18:56:59 +08:00
parent e3a059eb7c
commit e33dfd7992
30 changed files with 504 additions and 21 deletions

View File

@@ -1,5 +1,8 @@
package constants
const (
ActionTypeVisit = "visit"
ActionTypeVisit = "visit"
ActionTypeInstallDep = "install_dep"
ActionTypeInstallLang = "install_lang"
ActionTypeViewDisclaimer = "view_disclaimer"
)

View File

@@ -1,7 +1,20 @@
package constants
const (
ChallengeLogin7d = "login_7d"
ChallengeCreateCustomizedSpider = "create_customized_spider"
ChallengeRunRandom = "run_random"
ChallengeLogin7d = "login_7d"
ChallengeLogin30d = "login_30d"
ChallengeLogin90d = "login_90d"
ChallengeLogin180d = "login_180d"
ChallengeCreateCustomizedSpider = "create_customized_spider"
ChallengeCreateConfigurableSpider = "create_configurable_spider"
ChallengeCreateSchedule = "create_schedule"
ChallengeCreateNodes = "create_nodes"
ChallengeCreateUser = "create_user"
ChallengeRunRandom = "run_random"
ChallengeScrape1k = "scrape_1k"
ChallengeScrape10k = "scrape_10k"
ChallengeScrape100k = "scrape_100k"
ChallengeInstallDep = "install_dep"
ChallengeInstallLang = "install_lang"
ChallengeViewDisclaimer = "view_disclaimer"
)

View File

@@ -7,22 +7,136 @@
"description_en": "Logged-in for consecutive 7 days to complete the challenge",
"difficulty": 1
},
{
"name": "login_30d",
"title_cn": "连续登录 30 天",
"title_en": "Logged-in for 30 days",
"description_cn": "连续 30 天登录 Crawlab即可完成挑战",
"description_en": "Logged-in for consecutive 30 days to complete the challenge",
"difficulty": 2
},
{
"name": "login_90d",
"title_cn": "连续登录 90 天",
"title_en": "Logged-in for 90 days",
"description_cn": "连续 90 天登录 Crawlab即可完成挑战",
"description_en": "Logged-in for consecutive 90 days to complete the challenge",
"difficulty": 3
},
{
"name": "login_180d",
"title_cn": "连续登录 180 天",
"title_en": "Logged-in for 180 days",
"description_cn": "连续 180 天登录 Crawlab即可完成挑战",
"description_en": "Logged-in for consecutive 180 days to complete the challenge",
"difficulty": 4
},
{
"name": "create_customized_spider",
"title_cn": "创建个自定义爬虫",
"title_cn": "创建 1 个自定义爬虫",
"title_en": "Create a customized spider",
"description_cn": "在爬虫列表中,点击 '添加爬虫',选择 '自定义爬虫',输入相应的参数,点击添加,即可完成挑战!",
"description_en": "In Spider List page, click 'Add Spider', select 'Customized Spider', enter params, click 'Add' to finish the challenge.",
"difficulty": 1,
"path": "/spiders"
},
{
"name": "create_configurable_spider",
"title_cn": "创建 1 个可配置爬虫",
"title_en": "Create a configurable spider",
"description_cn": "在爬虫列表中,点击 '添加爬虫',选择 '可配置爬虫',输入相应的参数,点击添加,即可完成挑战!",
"description_en": "In Spider List page, click 'Add Spider', select 'Configurable Spider', enter params, click 'Add' to finish the challenge.",
"difficulty": 1,
"path": "/spiders"
},
{
"name": "run_random",
"title_cn": "用随机模式运行一个爬虫",
"title_en": "Run a spider in random mode",
"description_cn": "在您创建好的爬虫中,导航到其对应的详情页(爬虫列表中点击爬虫),运行一个爬虫,选择随机模式。",
"description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode.",
"title_cn": "用随机模式成功运行爬虫",
"title_en": "Run a spider in random mode successfully",
"description_cn": "在您创建好的爬虫中,导航到其对应的详情页(爬虫列表中点击爬虫),选择随机模式运行一个爬虫,并能运行成功。",
"description_en": "In your created spiders, navigate to corresponding detail page (click spider in Spider List page), run a spider in random mode successfully.",
"difficulty": 1,
"path": "/spiders"
},
{
"name": "scrape_1k",
"title_cn": "抓取 1 千条数据",
"title_en": "Scrape 1k records",
"description_cn": "运行您创建好的爬虫,抓取 1 千条及以上的结果数据,即可完成挑战!",
"description_en": "Run your created spiders, scrape 1k and more results to finish the challenge.",
"difficulty": 2,
"path": "/spiders"
},
{
"name": "scrape_10k",
"title_cn": "抓取 1 万条数据",
"title_en": "Scrape 10k records",
"description_cn": "运行您创建好的爬虫,抓取 1 万条及以上的结果数据,即可完成挑战!",
"description_en": "Run your created spiders, scrape 10k and more results to finish the challenge.",
"difficulty": 3,
"path": "/spiders"
},
{
"name": "scrape_100k",
"title_cn": "抓取 10 万条数据",
"title_en": "Scrape 100k records",
"description_cn": "运行您创建好的爬虫,抓取 10 万条及以上的结果数据,即可完成挑战!",
"description_en": "Run your created spiders, scrape 100k and more results to finish the challenge.",
"difficulty": 4,
"path": "/spiders"
},
{
"name": "create_schedule",
"title_cn": "创建 1 个定时任务",
"title_en": "Create a schedule",
"description_cn": "在定时任务列表中,创建一个定时任务,正确设置好 Cron 表达式,即可完成挑战!",
"description_en": "In Schedule List page, create a schedule and configure cron expression to finish the task.",
"difficulty": 1,
"path": "/schedules"
},
{
"name": "create_nodes",
"title_cn": "创建 1 个节点集群",
"title_en": "Create a node cluster",
"description_cn": "按照文档的部署指南,部署含有 3 个节点的集群,即可完成挑战!",
"description_en": "Deploy a 3-node cluster according to the deployment guidance in documentation to finish the task.",
"difficulty": 3,
"path": "/nodes"
},
{
"name": "install_dep",
"title_cn": "安装 1 个依赖",
"title_en": "Install a dependency successfully",
"description_cn": "在 '节点列表->安装' 或 '节点详情->安装' 中,搜索并安装所需的 1 个依赖,即可完成挑战!",
"description_en": "In 'Node List -> Installation' or 'Node Detail -> Installation', search and install a dependency.",
"difficulty": 3,
"path": "/nodes"
},
{
"name": "install_lang",
"title_cn": "安装 1 个语言环境",
"title_en": "Install a language successfully",
"description_cn": "在 '节点列表->安装' 或 '节点详情->安装' 中,点击安装所需的 1 个语言环境,即可完成挑战!",
"description_en": "In 'Node List -> Installation' or 'Node Detail -> Installation', install a language.",
"difficulty": 3,
"path": "/nodes"
},
{
"name": "view_disclaimer",
"title_cn": "阅读免责声明",
"title_en": "View disclaimer",
"description_cn": "在左侧菜单栏,点击 '免责声明' 查看其内容,即可完成挑战!",
"description_en": "In the left side menu, click 'Disclaimer' and view its content to finish the challenge.",
"difficulty": 1,
"path": "/disclaimer"
},
{
"name": "create_user",
"title_cn": "创建 1 个用户",
"title_en": "Create a user",
"description_cn": "在用户管理页面中创建一个新用户,即可完成挑战!",
"description_en": "In User Admin page, create a new user to finish the challenge.",
"difficulty": 1,
"path": "/users"
}
]

View File

@@ -240,6 +240,7 @@ func main() {
authGroup.GET("/users/:id", routes.GetUser) // 用户详情
authGroup.POST("/users/:id", routes.PostUser) // 更改用户
authGroup.DELETE("/users/:id", routes.DeleteUser) // 删除用户
authGroup.PUT("/users-add", routes.PutUser) // 添加用户
authGroup.GET("/me", routes.GetMe) // 获取自己账户
authGroup.POST("/me", routes.PostMe) // 修改自己账户
}

View File

@@ -32,7 +32,7 @@ func (ch *Challenge) Save() error {
ch.UpdateTs = time.Now()
if err := c.UpdateId(ch.Id, c); err != nil {
if err := c.UpdateId(ch.Id, ch); err != nil {
debug.PrintStack()
return err
}

View File

@@ -26,7 +26,7 @@ type Task struct {
TotalDuration float64 `json:"total_duration" bson:"total_duration"`
Pid int `json:"pid" bson:"pid"`
RunType string `json:"run_type" bson:"run_type"`
ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"`
//ScheduleId bson.ObjectId `json:"schedule_id" bson:"schedule_id"`
// 前端数据
SpiderName string `json:"spider_name"`

View File

@@ -19,8 +19,9 @@ type User struct {
Email string `json:"email" bson:"email"`
Setting UserSetting `json:"setting" bson:"setting"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
UserId bson.ObjectId `json:"user_id" bson:"user_id"`
CreateTs time.Time `json:"create_ts" bson:"create_ts"`
UpdateTs time.Time `json:"update_ts" bson:"update_ts"`
}
type UserSetting struct {

View File

@@ -95,8 +95,11 @@ func PutUser(c *gin.Context) {
reqData.Role = constants.RoleNormal
}
// UserId
uid := services.GetCurrentUserId(c)
// 添加用户
if err := services.CreateNewUser(reqData.Username, reqData.Password, reqData.Role, reqData.Email); err != nil {
if err := services.CreateNewUser(reqData.Username, reqData.Password, reqData.Role, reqData.Email, uid); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
}
@@ -120,6 +123,10 @@ func PostUser(c *gin.Context) {
return
}
if item.UserId.Hex() == "" {
item.UserId = bson.ObjectIdHex(constants.ObjectIdNull)
}
if err := model.UpdateUser(bson.ObjectIdHex(id), item); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return
@@ -230,6 +237,11 @@ func PostMe(c *gin.Context) {
user.Setting.WechatRobotWebhook = reqBody.Setting.WechatRobotWebhook
}
user.Setting.EnabledNotifications = reqBody.Setting.EnabledNotifications
if user.UserId.Hex() == "" {
user.UserId = bson.ObjectIdHex(constants.ObjectIdNull)
}
if err := user.Save(); err != nil {
HandleError(http.StatusInternalServerError, c, err)
return

View File

@@ -19,10 +19,36 @@ func GetService(name string, uid bson.ObjectId) Service {
switch name {
case constants.ChallengeLogin7d:
return &Login7dService{UserId: uid}
case constants.ChallengeLogin30d:
return &Login30dService{UserId: uid}
case constants.ChallengeLogin90d:
return &Login90dService{UserId: uid}
case constants.ChallengeLogin180d:
return &Login180dService{UserId: uid}
case constants.ChallengeCreateCustomizedSpider:
return &CreateCustomizedSpiderService{UserId: uid}
case constants.ChallengeCreateConfigurableSpider:
return &CreateConfigurableSpiderService{UserId: uid}
case constants.ChallengeCreateSchedule:
return &CreateScheduleService{UserId: uid}
case constants.ChallengeCreateNodes:
return &CreateNodesService{UserId: uid}
case constants.ChallengeRunRandom:
return &RunRandomService{UserId: uid}
case constants.ChallengeScrape1k:
return &Scrape1kService{UserId: uid}
case constants.ChallengeScrape10k:
return &Scrape10kService{UserId: uid}
case constants.ChallengeScrape100k:
return &Scrape100kService{UserId: uid}
case constants.ChallengeInstallDep:
return &InstallDepService{UserId: uid}
case constants.ChallengeInstallLang:
return &InstallLangService{UserId: uid}
case constants.ChallengeViewDisclaimer:
return &ViewDisclaimerService{UserId: uid}
case constants.ChallengeCreateUser:
return &CreateUserService{UserId: uid}
}
return nil
}
@@ -98,6 +124,8 @@ func InitChallengeService() error {
continue
}
} else {
ch.Id = chDb.Id
ch.CreateTs = chDb.CreateTs
if err := ch.Save(); err != nil {
log.Errorf(err.Error())
debug.PrintStack()

View File

@@ -0,0 +1,23 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type CreateConfigurableSpiderService struct {
UserId bson.ObjectId
}
func (s *CreateConfigurableSpiderService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"type": constants.Configurable,
}
_, count, err := model.GetSpiderList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return count > 0, nil
}

View File

@@ -0,0 +1,22 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type CreateNodesService struct {
UserId bson.ObjectId
}
func (s *CreateNodesService) Check() (bool, error) {
query := bson.M{
"status": constants.StatusOnline,
}
list, err := model.GetScheduleList(query)
if err != nil {
return false, err
}
return len(list) >= 3, nil
}

View File

@@ -0,0 +1,21 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type CreateScheduleService struct {
UserId bson.ObjectId
}
func (s *CreateScheduleService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
}
list, err := model.GetScheduleList(query)
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,21 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type CreateUserService struct {
UserId bson.ObjectId
}
func (s *CreateUserService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
}
list, err := model.GetUserList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,23 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type InstallDepService struct {
UserId bson.ObjectId
}
func (s *InstallDepService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"type": constants.ActionTypeInstallDep,
}
list, err := model.GetActionList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,23 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type InstallLangService struct {
UserId bson.ObjectId
}
func (s *InstallLangService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"type": constants.ActionTypeInstallLang,
}
list, err := model.GetActionList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,18 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Login180dService struct {
UserId bson.ObjectId
}
func (s *Login180dService) Check() (bool, error) {
days, err := model.GetVisitDays(s.UserId)
if err != nil {
return false, err
}
return days >= 180, nil
}

View File

@@ -0,0 +1,18 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Login30dService struct {
UserId bson.ObjectId
}
func (s *Login30dService) Check() (bool, error) {
days, err := model.GetVisitDays(s.UserId)
if err != nil {
return false, err
}
return days >= 30, nil
}

View File

@@ -0,0 +1,18 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Login90dService struct {
UserId bson.ObjectId
}
func (s *Login90dService) Check() (bool, error) {
days, err := model.GetVisitDays(s.UserId)
if err != nil {
return false, err
}
return days >= 90, nil
}

View File

@@ -14,6 +14,7 @@ func (s *RunRandomService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"run_type": constants.RunTypeRandom,
"status": constants.StatusFinished,
}
list, err := model.GetTaskList(query, 0, 1, "-_id")
if err != nil {

View File

@@ -0,0 +1,24 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Scrape100kService struct {
UserId bson.ObjectId
}
func (s *Scrape100kService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"result_count": bson.M{
"$gte": 100000,
},
}
list, err := model.GetTaskList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,24 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Scrape10kService struct {
UserId bson.ObjectId
}
func (s *Scrape10kService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"result_count": bson.M{
"$gte": 10000,
},
}
list, err := model.GetTaskList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,24 @@
package challenge
import (
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type Scrape1kService struct {
UserId bson.ObjectId
}
func (s *Scrape1kService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"result_count": bson.M{
"$gte": 1000,
},
}
list, err := model.GetTaskList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -0,0 +1,23 @@
package challenge
import (
"crawlab/constants"
"crawlab/model"
"github.com/globalsign/mgo/bson"
)
type ViewDisclaimerService struct {
UserId bson.ObjectId
}
func (s *ViewDisclaimerService) Check() (bool, error) {
query := bson.M{
"user_id": s.UserId,
"type": constants.ActionTypeViewDisclaimer,
}
list, err := model.GetActionList(query, 0, 1, "-_id")
if err != nil {
return false, err
}
return len(list) > 0, nil
}

View File

@@ -14,7 +14,7 @@ import (
)
func InitUserService() error {
_ = CreateNewUser("admin", "admin", constants.RoleAdmin, "")
_ = CreateNewUser("admin", "admin", constants.RoleAdmin, "", bson.ObjectIdHex(constants.ObjectIdNull))
return nil
}
@@ -90,12 +90,13 @@ func CheckToken(tokenStr string) (user model.User, err error) {
return
}
func CreateNewUser(username string, password string, role string, email string) error {
func CreateNewUser(username string, password string, role string, email string, uid bson.ObjectId) error {
user := model.User{
Username: strings.ToLower(username),
Password: utils.EncryptPassword(password),
Role: role,
Email: email,
UserId: uid,
Setting: model.UserSetting{
NotificationTrigger: constants.NotificationTriggerNever,
EnabledNotifications: []string{
@@ -112,11 +113,10 @@ func CreateNewUser(username string, password string, role string, email string)
}
func GetCurrentUser(c *gin.Context) *model.User {
data, _ := c.Get("currentUser")
data, _ := c.Get(constants.ContextUser)
return data.(*model.User)
}
func GetCurrentUserId(c *gin.Context) bson.ObjectId {
return GetCurrentUser(c).Id
}

View File

@@ -262,6 +262,9 @@ export default {
})
dep.installed = true
}
this.$request.put('/actions', {
type: 'install_dep'
})
this.$set(this.depLoadingDict, name, false)
this.$st.sendEv('节点详情', '安装', '安装依赖')
},
@@ -312,6 +315,9 @@ export default {
message: this.$t('You have successfully installed a language: ') + this.activeLang.name
})
}
this.$request.put('/actions', {
type: 'install_lang'
})
this.isLoadingInstallLang = false
this.$st.sendEv('节点详情', '安装', '安装语言')
},

View File

@@ -266,6 +266,7 @@ export default {
await Promise.all(this.nodeList.map(async n => {
if (n.status !== 'online') return
const res = await this.$request.get(`/nodes/${n._id}/langs`)
if (!res.data.data) return
res.data.data.forEach(l => {
const key = n._id + '|' + l.executable_name
this.$set(this.langsDataDict, key, l)
@@ -280,6 +281,7 @@ export default {
await Promise.all(this.nodeList.map(async n => {
if (n.status !== 'online') return
const res = await this.$request.get(`/nodes/${n._id}/deps/installed`, { lang: this.activeLang })
if (!res.data.data) return
res.data.data.forEach(d => {
depsSet.add(d.name)
const key = n._id + '|' + d.name
@@ -319,6 +321,9 @@ export default {
setTimeout(() => {
this.getLangsData()
}, 1000)
this.$request.put('/actions', {
type: 'install_lang'
})
this.$st.sendEv('节点列表', '安装', '安装语言')
},
async onInstallLangAll (langLabel, ev) {
@@ -372,6 +377,9 @@ export default {
})
this.$set(this.depsDataDict, key, 'installed')
}
this.$request.put('/actions', {
type: 'install_dep'
})
this.$st.sendEv('节点列表', '安装', '安装依赖')
},
async uninstallDep (node, dep) {

View File

@@ -592,6 +592,7 @@ docker run -d --restart always --name crawlab_worker \\
'Are you sure to stop selected items?': '您是否确认停止所选项',
'Sent signals to cancel selected tasks': '已经向所选任务发送取消任务信号',
'Copied successfully': '已成功复制',
'You have started the challenge.': '您已开始挑战',
// 其他
'Star crawlab-team/crawlab on GitHub': ' GitHub 上为 Crawlab 加星吧'

View File

@@ -152,7 +152,7 @@ const user = {
// 添加用户
addUser ({ dispatch, commit, state }) {
return request.put('/users', state.userForm)
return request.put('/users-add', state.userForm)
},
// 新增全局变量
addGlobalVariable ({ commit, state }) {

View File

@@ -49,6 +49,7 @@
size="mini"
type="primary"
icon="el-icon-s-flag"
@click="onStartChallenge(c)"
>
{{$t('Start Challenge')}}
</el-button>
@@ -80,6 +81,13 @@ export default {
await this.$request.post('/challenges-check')
const res = await this.$request.get('/challenges')
this.challenges = res.data.data || []
},
onStartChallenge (c) {
if (c.path) {
this.$router.push(c.path)
} else {
this.$message.success(this.$t('You have started the challenge.'))
}
}
},
async created () {
@@ -96,8 +104,8 @@ export default {
}
.challenge-list .challenge-item {
flex-basis: 240px;
width: 240px;
flex-basis: 280px;
width: 280px;
margin: 10px;
}

View File

@@ -60,6 +60,11 @@ This Disclaimer and privacy protection statement (hereinafter referred to as "di
8. 传播:任何公司或个人在网络上发布,传播我们软件的行为都是允许的,但因公司或个人传播软件可能造成的任何法律和刑事事件 Crawlab 开发组不负任何责任。
`
}
},
mounted () {
this.$request.put('/actions', {
type: 'view_disclaimer'
})
}
}
</script>