+
|
+ |||||||||||
{{ $line }}
+ {{ end }} +{{ end }} +{{ if (ne .Email.Body.FreeMarkdown "") }} + {{ .Email.Body.FreeMarkdown.ToHTML }} +{{ else }} + {{ with .Email.Body.Dictionary }} +-
+ {{ range $entry := . }}
+
- {{ $entry.Key }}: {{ $entry.Value }} + {{ end }} +
| {{ $entry.Key }} | + {{ end }} +
|---|
| + {{ $cell.Value }} + | + {{ end }} +
{{ $action.Instructions }} {{ $action.Button.Link }}
+ {{ end }} + {{ end }} +{{ end }} +{{ with .Email.Body.Outros }} + {{ range $line := . }} +{{ $line }}
+ {{ end }} +{{ end }} +
{{.Email.Body.Signature}},
{{.Hermes.Product.Name}} - {{.Hermes.Product.Link}}
{{.Hermes.Product.Copyright}}
+` +} + +func (dt *MailThemeFlat) GetStyle() string { + return ` + +` +} diff --git a/core/notification/mobile.go b/core/notification/mobile.go new file mode 100644 index 00000000..cd36641b --- /dev/null +++ b/core/notification/mobile.go @@ -0,0 +1,62 @@ +package notification + +import ( + "errors" + "github.com/crawlab-team/go-trace" + "github.com/imroc/req" + "strings" +) + +type ResBody struct { + ErrCode int `json:"errcode"` + ErrMsg string `json:"errmsg"` +} + +func SendMobileNotification(webhook string, title string, content string) error { + // request header + header := req.Header{ + "Content-Type": "application/json; charset=utf-8", + } + + // request data + data := req.Param{ + "msgtype": "markdown", + "markdown": req.Param{ + "title": title, + "text": content, + "content": content, + }, + "at": req.Param{ + "atMobiles": []string{}, + "isAtAll": false, + }, + "text": content, + } + if strings.Contains(strings.ToLower(webhook), "feishu") { + data = req.Param{ + "msg_type": "text", + "content": req.Param{ + "text": content, + }, + } + } + + // perform request + res, err := req.Post(webhook, header, req.BodyJSON(&data)) + if err != nil { + return trace.TraceError(err) + } + + // parse response + var resBody ResBody + if err := res.ToJSON(&resBody); err != nil { + return trace.TraceError(err) + } + + // validate response code + if resBody.ErrCode != 0 { + return errors.New(resBody.ErrMsg) + } + + return nil +} diff --git a/core/notification/models.go b/core/notification/models.go new file mode 100644 index 00000000..efb55448 --- /dev/null +++ b/core/notification/models.go @@ -0,0 +1,32 @@ +package notification + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type NotificationSetting struct { + Id primitive.ObjectID `json:"_id" bson:"_id"` + Type string `json:"type" bson:"type"` + Name string `json:"name" bson:"name"` + Description string `json:"description" bson:"description"` + Enabled bool `json:"enabled" bson:"enabled"` + Global bool `json:"global" bson:"global"` + Title string `json:"title,omitempty" bson:"title,omitempty"` + Template string `json:"template,omitempty" bson:"template,omitempty"` + TaskTrigger string `json:"task_trigger" bson:"task_trigger"` + Mail NotificationSettingMail `json:"mail,omitempty" bson:"mail,omitempty"` + Mobile NotificationSettingMobile `json:"mobile,omitempty" bson:"mobile,omitempty"` +} + +type NotificationSettingMail struct { + Server string `json:"server" bson:"server"` + Port string `json:"port,omitempty" bson:"port,omitempty"` + User string `json:"user,omitempty" bson:"user,omitempty"` + Password string `json:"password,omitempty" bson:"password,omitempty"` + SenderEmail string `json:"sender_email,omitempty" bson:"sender_email,omitempty"` + SenderIdentity string `json:"sender_identity,omitempty" bson:"sender_identity,omitempty"` + To string `json:"to,omitempty" bson:"to,omitempty"` + Cc string `json:"cc,omitempty" bson:"cc,omitempty"` +} + +type NotificationSettingMobile struct { + Webhook string `json:"webhook" bson:"webhook"` +} diff --git a/core/notification/payload.go b/core/notification/payload.go new file mode 100644 index 00000000..7337ebe0 --- /dev/null +++ b/core/notification/payload.go @@ -0,0 +1,8 @@ +package notification + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type SendPayload struct { + TaskId primitive.ObjectID `json:"task_id"` + Data string `json:"data"` +} diff --git a/core/notification/service.go b/core/notification/service.go new file mode 100644 index 00000000..b2fd8a27 --- /dev/null +++ b/core/notification/service.go @@ -0,0 +1,395 @@ +package notification + +import ( + "github.com/apex/log" + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + parser "github.com/crawlab-team/template-parser" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type Service struct { + col *mongo2.Col // notification settings + modelSvc service.ModelService +} + +func (svc *Service) Init() (err error) { + if !utils.IsPro() { + return nil + } + + return nil +} + +func (svc *Service) Start() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *Service) Stop() (err error) { + return nil +} + +func (svc *Service) initData() (err error) { + total, err := svc.col.Count(nil) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []NotificationSetting{ + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "任务通知（邮件）", + Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 爬虫任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +|键|值| +|:-:|:--| +|任务状态|{{$.status}}| +|任务优先级|{{$.priority}}| +|任务模式|{{$.mode}}| +|执行命令|{{$.cmd}}| +|执行参数|{{$.param}}| +|错误信息|{{$.error}}| +|节点|{{$.node.name}}| +|爬虫|{{$.spider.name}}| +|项目|{{$.spider.project.name}}| +|定时任务|{{$.schedule.name}}| +|结果数|{{$.:task_stat.result_count}}| +|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| +|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| +|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| +|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "Task Change (Mail)", + Description: "This is the default mail notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +|Key|Value| +|:-:|:--| +|Task Status|{{$.status}}| +|Task Priority|{{$.priority}}| +|Task Mode|{{$.mode}}| +|Task Command|{{$.cmd}}| +|Task Params|{{$.param}}| +|Error Message|{{$.error}}| +|Node|{{$.node.name}}| +|Spider|{{$.spider.name}}| +|Project|{{$.spider.project.name}}| +|Schedule|{{$.schedule.name}}| +|Result Count|{{$.:task_stat.result_count}}| +|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| +|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| +|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| +|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "任务通知（移动端）", + Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +- **任务状态**: {{$.status}} +- **任务优先级**: {{$.priority}} +- **任务模式**: {{$.mode}} +- **执行命令**: {{$.cmd}} +- **执行参数**: {{$.param}} +- **错误信息**: {{$.error}} +- **节点**: {{$.node.name}} +- **爬虫**: {{$.spider.name}} +- **项目**: {{$.spider.project.name}} +- **定时任务**: {{$.schedule.name}} +- **结果数**: {{$.:task_stat.result_count}} +- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} +- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} +- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: NotificationSettingMobile{}, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "Task Change (Mobile)", + Description: "This is the default mobile notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskError, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +- **Task Status**: {{$.status}} +- **Task Priority**: {{$.priority}} +- **Task Mode**: {{$.mode}} +- **Task Command**: {{$.cmd}} +- **Task Params**: {{$.param}} +- **Error Message**: {{$.error}} +- **Node**: {{$.node.name}} +- **Spider**: {{$.spider.name}} +- **Project**: {{$.spider.project.name}} +- **Schedule**: {{$.schedule.name}} +- **Result Count**: {{$.:task_stat.result_count}} +- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} +- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} +- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: NotificationSettingMobile{}, + }, + } + var data []interface{} + for _, s := range settings { + data = append(data, s) + } + _, err = svc.col.InsertMany(data) + if err != nil { + return err + } + return nil +} + +func (svc *Service) Send(s NotificationSetting, entity bson.M) (err error) { + switch s.Type { + case TypeMail: + return svc.SendMail(s, entity) + case TypeMobile: + return svc.SendMobile(s, entity) + } + return nil +} + +func (svc *Service) SendMail(s NotificationSetting, entity bson.M) (err error) { + // to + to, err := parser.Parse(s.Mail.To, entity) + if err != nil { + log.Warnf("parsing 'to' error: %v", err) + } + if to == "" { + return nil + } + + // cc + cc, err := parser.Parse(s.Mail.Cc, entity) + if err != nil { + log.Warnf("parsing 'cc' error: %v", err) + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send mail + if err := SendMail(&models.NotificationSettingMail{ + Server: s.Mail.Server, + Port: s.Mail.Port, + User: s.Mail.User, + Password: s.Mail.Password, + SenderEmail: s.Mail.SenderEmail, + SenderIdentity: s.Mail.SenderIdentity, + To: s.Mail.To, + Cc: s.Mail.Cc, + }, to, cc, title, content); err != nil { + return err + } + + return nil +} + +func (svc *Service) SendMobile(s NotificationSetting, entity bson.M) (err error) { + // webhook + webhook, err := parser.Parse(s.Mobile.Webhook, entity) + if err != nil { + log.Warnf("parsing 'webhook' error: %v", err) + } + if webhook == "" { + return nil + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send + if err := SendMobileNotification(webhook, title, content); err != nil { + return err + } + + return nil +} + +func (svc *Service) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []NotificationSetting, total int, err error) { + // options + var options *mongo2.FindOptions + if pagination != nil || sort != nil { + options = new(mongo2.FindOptions) + if pagination != nil { + options.Skip = pagination.Size * (pagination.Page - 1) + options.Limit = pagination.Size + } + if sort != nil { + options.Sort = sort + } + } + + // get list + var list []NotificationSetting + if err := svc.col.Find(query, options).All(&list); err != nil { + if err.Error() == mongo.ErrNoDocuments.Error() { + return nil, 0, nil + } else { + return nil, 0, err + } + } + + // total count + total, err = svc.col.Count(query) + if err != nil { + return nil, 0, err + } + + return list, total, nil +} + +func (svc *Service) GetSetting(id primitive.ObjectID) (res *NotificationSetting, err error) { + var s NotificationSetting + if err := svc.col.FindId(id).One(&s); err != nil { + return nil, err + } + return &s, nil +} + +func (svc *Service) PosSetting(s *NotificationSetting) (err error) { + s.Id = primitive.NewObjectID() + if _, err := svc.col.Insert(s); err != nil { + return err + } + return nil +} + +func (svc *Service) PutSetting(id primitive.ObjectID, s NotificationSetting) (err error) { + if err := svc.col.ReplaceId(id, s); err != nil { + return err + } + + return nil +} + +func (svc *Service) DeleteSetting(id primitive.ObjectID) (err error) { + if err := svc.col.DeleteId(id); err != nil { + return err + } + + return nil +} + +func (svc *Service) EnableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(true)(id) +} + +func (svc *Service) DisableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(false)(id) +} + +func (svc *Service) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { + return func(id primitive.ObjectID) (err error) { + var s NotificationSetting + if err := svc.col.FindId(id).One(&s); err != nil { + return err + } + s.Enabled = value + if err := svc.col.ReplaceId(id, s); err != nil { + return err + } + return nil + } +} + +func NewService() *Service { + // service + svc := &Service{ + col: mongo2.GetMongoCol(SettingsColName), + } + + // model service + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + svc.modelSvc = modelSvc + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _service *Service + +func GetService() *Service { + if _service == nil { + _service = NewService() + } + return _service +} diff --git a/core/notification/service_test.go b/core/notification/service_test.go new file mode 100644 index 00000000..20e09478 --- /dev/null +++ b/core/notification/service_test.go @@ -0,0 +1,19 @@ +package notification + +import ( + "net/http" + "testing" + "time" +) + +func TestService_sendMobile(t *testing.T) { + T.Setup(t) + e := T.NewExpect(t) + time.Sleep(1 * time.Second) + + data := map[string]interface{}{ + "task_id": T.TestTask.GetId().Hex(), + } + e.POST("/send/mobile").WithJSON(data). + Expect().Status(http.StatusOK) +} diff --git a/core/notification/service_v2.go b/core/notification/service_v2.go new file mode 100644 index 00000000..23ade59b --- /dev/null +++ b/core/notification/service_v2.go @@ -0,0 +1,362 @@ +package notification + +import ( + "github.com/apex/log" + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + parser "github.com/crawlab-team/template-parser" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type ServiceV2 struct { +} + +func (svc *ServiceV2) Start() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) Stop() (err error) { + return nil +} + +func (svc *ServiceV2) initData() (err error) { + total, err := service.NewModelServiceV2[models.NotificationSettingV2]().Count(nil) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.NotificationSettingV2{ + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "任务通知（邮件）", + Description: "这是默认的邮件通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 爬虫任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +|键|值| +|:-:|:--| +|任务状态|{{$.status}}| +|任务优先级|{{$.priority}}| +|任务模式|{{$.mode}}| +|执行命令|{{$.cmd}}| +|执行参数|{{$.param}}| +|错误信息|{{$.error}}| +|节点|{{$.node.name}}| +|爬虫|{{$.spider.name}}| +|项目|{{$.spider.project.name}}| +|定时任务|{{$.schedule.name}}| +|结果数|{{$.:task_stat.result_count}}| +|等待时间（秒）|{#{{$.:task_stat.wait_duration}}/1000#}| +|运行时间（秒）|{#{{$.:task_stat.runtime_duration}}/1000#}| +|总时间（秒）|{#{{$.:task_stat.total_duration}}/1000#}| +|平均结果数/秒|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: models.NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMail, + Enabled: true, + Name: "Task Change (Mail)", + Description: "This is the default mail notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +|Key|Value| +|:-:|:--| +|Task Status|{{$.status}}| +|Task Priority|{{$.priority}}| +|Task Mode|{{$.mode}}| +|Task Command|{{$.cmd}}| +|Task Params|{{$.param}}| +|Error Message|{{$.error}}| +|Node|{{$.node.name}}| +|Spider|{{$.spider.name}}| +|Project|{{$.spider.project.name}}| +|Schedule|{{$.schedule.name}}| +|Result Count|{{$.:task_stat.result_count}}| +|Wait Duration (sec)|{#{{$.:task_stat.wait_duration}}/1000#}| +|Runtime Duration (sec)|{#{{$.:task_stat.runtime_duration}}/1000#}| +|Total Duration (sec)|{#{{$.:task_stat.total_duration}}/1000#}| +|Avg Results / Sec|{#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}| +`, + Mail: models.NotificationSettingMail{ + Server: "smtp.163.com", + Port: "465", + To: "{{$.user[create].email}}", + }, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "任务通知（移动端）", + Description: "这是默认的手机通知。您可以使用您自己的设置进行编辑。", + TaskTrigger: constants.NotificationTriggerTaskFinish, + Title: "[Crawlab] 任务更新: {{$.status}}", + Template: `尊敬的 {{$.user.username}}, + +请查看下面的任务数据。 + +- **任务状态**: {{$.status}} +- **任务优先级**: {{$.priority}} +- **任务模式**: {{$.mode}} +- **执行命令**: {{$.cmd}} +- **执行参数**: {{$.param}} +- **错误信息**: {{$.error}} +- **节点**: {{$.node.name}} +- **爬虫**: {{$.spider.name}} +- **项目**: {{$.spider.project.name}} +- **定时任务**: {{$.schedule.name}} +- **结果数**: {{$.:task_stat.result_count}} +- **等待时间（秒）**: {#{{$.:task_stat.wait_duration}}/1000#} +- **运行时间（秒）**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **总时间（秒）**: {#{{$.:task_stat.total_duration}}/1000#} +- **平均结果数/秒**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: models.NotificationSettingMobile{}, + }, + { + Id: primitive.NewObjectID(), + Type: TypeMobile, + Enabled: true, + Name: "Task Change (Mobile)", + Description: "This is the default mobile notification. You can edit it with your own settings", + TaskTrigger: constants.NotificationTriggerTaskError, + Title: "[Crawlab] Task Update: {{$.status}}", + Template: `Dear {{$.user.username}}, + +Please find the task data as below. + +- **Task Status**: {{$.status}} +- **Task Priority**: {{$.priority}} +- **Task Mode**: {{$.mode}} +- **Task Command**: {{$.cmd}} +- **Task Params**: {{$.param}} +- **Error Message**: {{$.error}} +- **Node**: {{$.node.name}} +- **Spider**: {{$.spider.name}} +- **Project**: {{$.spider.project.name}} +- **Schedule**: {{$.schedule.name}} +- **Result Count**: {{$.:task_stat.result_count}} +- **Wait Duration (sec)**: {#{{$.:task_stat.wait_duration}}/1000#} +- **Runtime Duration (sec)**: {#{{$.:task_stat.runtime_duration}}/1000#} +- **Total Duration (sec)**: {#{{$.:task_stat.total_duration}}/1000#} +- **Avg Results / Sec**: {#{{$.:task_stat.result_count}}/({{$.:task_stat.total_duration}}/1000)#}`, + Mobile: models.NotificationSettingMobile{}, + }, + } + _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertMany(settings) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) Send(s *models.NotificationSettingV2, entity bson.M) (err error) { + switch s.Type { + case TypeMail: + return svc.SendMail(s, entity) + case TypeMobile: + return svc.SendMobile(s, entity) + } + return nil +} + +func (svc *ServiceV2) SendMail(s *models.NotificationSettingV2, entity bson.M) (err error) { + // to + to, err := parser.Parse(s.Mail.To, entity) + if err != nil { + log.Warnf("parsing 'to' error: %v", err) + } + if to == "" { + return nil + } + + // cc + cc, err := parser.Parse(s.Mail.Cc, entity) + if err != nil { + log.Warnf("parsing 'cc' error: %v", err) + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send mail + if err := SendMail(&s.Mail, to, cc, title, content); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) SendMobile(s *models.NotificationSettingV2, entity bson.M) (err error) { + // webhook + webhook, err := parser.Parse(s.Mobile.Webhook, entity) + if err != nil { + log.Warnf("parsing 'webhook' error: %v", err) + } + if webhook == "" { + return nil + } + + // title + title, err := parser.Parse(s.Title, entity) + if err != nil { + log.Warnf("parsing 'title' error: %v", err) + } + + // content + content, err := parser.Parse(s.Template, entity) + if err != nil { + log.Warnf("parsing 'content' error: %v", err) + } + + // send + if err := SendMobileNotification(webhook, title, content); err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) GetSettingList(query bson.M, pagination *entity.Pagination, sort bson.D) (res []models.NotificationSettingV2, total int, err error) { + // options + var options *mongo2.FindOptions + if pagination != nil || sort != nil { + options = new(mongo2.FindOptions) + if pagination != nil { + options.Skip = pagination.Size * (pagination.Page - 1) + options.Limit = pagination.Size + } + if sort != nil { + options.Sort = sort + } + } + + // get list + list, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetMany(query, options) + if err != nil { + if err.Error() == mongo.ErrNoDocuments.Error() { + return nil, 0, nil + } else { + return nil, 0, err + } + } + + // total count + total, err = service.NewModelServiceV2[models.NotificationSettingV2]().Count(query) + if err != nil { + return nil, 0, err + } + + return list, total, nil +} + +func (svc *ServiceV2) GetSetting(id primitive.ObjectID) (res *models.NotificationSettingV2, err error) { + s, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetById(id) + if err != nil { + return nil, err + } + return s, nil +} + +func (svc *ServiceV2) PosSetting(s *models.NotificationSettingV2) (err error) { + s.Id = primitive.NewObjectID() + _, err = service.NewModelServiceV2[models.NotificationSettingV2]().InsertOne(*s) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) PutSetting(id primitive.ObjectID, s models.NotificationSettingV2) (err error) { + err = service.NewModelServiceV2[models.NotificationSettingV2]().ReplaceById(id, s) + if err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) DeleteSetting(id primitive.ObjectID) (err error) { + err = service.NewModelServiceV2[models.NotificationSettingV2]().DeleteById(id) + if err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) EnableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(true)(id) +} + +func (svc *ServiceV2) DisableSetting(id primitive.ObjectID) (err error) { + return svc._toggleSettingFunc(false)(id) +} + +func (svc *ServiceV2) _toggleSettingFunc(value bool) func(id primitive.ObjectID) error { + return func(id primitive.ObjectID) (err error) { + s, err := service.NewModelServiceV2[models.NotificationSettingV2]().GetById(id) + if err != nil { + return err + } + s.Enabled = value + err = service.NewModelServiceV2[models.NotificationSettingV2]().ReplaceById(id, *s) + if err != nil { + return err + } + return nil + } +} + +func NewServiceV2() *ServiceV2 { + // service + svc := &ServiceV2{} + + return svc +} + +var _serviceV2 *ServiceV2 + +func GetServiceV2() *ServiceV2 { + if _serviceV2 == nil { + _serviceV2 = NewServiceV2() + } + return _serviceV2 +} diff --git a/core/process/daemon.go b/core/process/daemon.go new file mode 100644 index 00000000..0a5395c4 --- /dev/null +++ b/core/process/daemon.go @@ -0,0 +1,170 @@ +package process + +import ( + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/go-trace" + "math/rand" + "os/exec" + "time" +) + +const ( + SignalCreate = iota + SignalStart + SignalStopped + SignalError + SignalExited + SignalReachedMaxErrors +) + +type Daemon struct { + // settings + maxErrors int + exitTimeout time.Duration + + // internals + errors int + errMsg string + exitCode int + newCmdFn func() *exec.Cmd + cmd *exec.Cmd + stopped bool + ch chan int +} + +func (d *Daemon) Start() (err error) { + go d.handleSignal() + + for { + // command + d.cmd = d.newCmdFn() + d.ch <- SignalCreate + + // attempt to run + _ = d.cmd.Start() + d.ch <- SignalStart + + if err := d.cmd.Wait(); err != nil { + // stopped + d.ch <- SignalStopped + if d.stopped { + log.Infof("daemon stopped") + return nil + } + + // error + d.ch <- SignalError + d.errMsg = err.Error() + trace.PrintError(err) + } + + // exited + d.ch <- SignalExited + + // exit code + d.exitCode = d.cmd.ProcessState.ExitCode() + + // check exit code + if d.exitCode == 0 { + log.Infof("process exited with code 0") + return + } + + // error message + d.errMsg = errors.ErrorProcessDaemonProcessExited.Error() + + // increment errors + d.errors++ + + // validate if error count exceeds max errors + if d.errors >= d.maxErrors { + log.Infof("reached max errors: %d", d.maxErrors) + d.ch <- SignalReachedMaxErrors + return errors.ErrorProcessReachedMaxErrors + } + + // re-attempt + waitSec := rand.Intn(5) + log.Infof("re-attempt to start process in %d seconds...", waitSec) + time.Sleep(time.Duration(waitSec) * time.Second) + } +} + +func (d *Daemon) Stop() { + d.stopped = true + opts := &sys_exec.KillProcessOptions{ + Timeout: d.exitTimeout, + Force: false, + } + _ = sys_exec.KillProcess(d.cmd, opts) +} + +func (d *Daemon) GetMaxErrors() (maxErrors int) { + return d.maxErrors +} + +func (d *Daemon) SetMaxErrors(maxErrors int) { + d.maxErrors = maxErrors +} + +func (d *Daemon) GetExitTimeout() (timeout time.Duration) { + return d.exitTimeout +} + +func (d *Daemon) SetExitTimeout(timeout time.Duration) { + d.exitTimeout = timeout +} + +func (d *Daemon) GetCmd() (cmd *exec.Cmd) { + return d.cmd +} + +func (d *Daemon) GetCh() (ch chan int) { + return d.ch +} + +func (d *Daemon) handleSignal() { + for { + select { + case signal := <-d.ch: + switch signal { + case SignalCreate: + log.Infof("process created") + case SignalStart: + log.Infof("process started") + case SignalStopped: + log.Infof("process stopped") + case SignalError: + trace.PrintError(errors.NewProcessError(d.errMsg)) + case SignalExited: + log.Infof("process exited") + case SignalReachedMaxErrors: + log.Infof("reached max errors") + return + } + } + } +} + +func NewProcessDaemon(newCmdFn func() *exec.Cmd, opts ...DaemonOption) (d interfaces.ProcessDaemon) { + // daemon + d = &Daemon{ + maxErrors: 5, + exitTimeout: 15 * time.Second, + errors: 0, + errMsg: "", + newCmdFn: newCmdFn, + stopped: false, + ch: make(chan int), + } + + // apply options + for _, opt := range opts { + opt(d) + } + + return d +} diff --git a/core/process/daemon_test.go b/core/process/daemon_test.go new file mode 100644 index 00000000..7734eb59 --- /dev/null +++ b/core/process/daemon_test.go @@ -0,0 +1,21 @@ +package process + +import ( + "github.com/stretchr/testify/require" + "os/exec" + "testing" +) + +func TestDaemon(t *testing.T) { + d := NewProcessDaemon(func() *exec.Cmd { + return exec.Command("echo", "hello") + }) + err := d.Start() + require.Nil(t, err) + + d = NewProcessDaemon(func() *exec.Cmd { + return exec.Command("return", "1") + }) + err = d.Start() + require.NotNil(t, err) +} diff --git a/core/process/manage.go b/core/process/manage.go new file mode 100644 index 00000000..d2b05013 --- /dev/null +++ b/core/process/manage.go @@ -0,0 +1,60 @@ +package process + +import ( + "github.com/crawlab-team/go-trace" + "os/exec" + "regexp" + "runtime" + "strings" +) + +var pidRegexp, _ = regexp.Compile("(?:^|\\s+)\\d+(?:$|\\s+)") + +func ProcessIdExists(id int) (ok bool) { + lines, err := ListProcess(string(rune(id))) + if err != nil { + return false + } + for _, line := range lines { + matched := pidRegexp.MatchString(line) + if matched { + return true + } + } + return false +} + +func ListProcess(text string) (lines []string, err error) { + if runtime.GOOS == "windows" { + return listProcessWindow(text) + } else { + return listProcessLinuxMac(text) + } +} + +func listProcessWindow(text string) (lines []string, err error) { + cmd := exec.Command("tasklist", "/fi", text) + out, err := cmd.CombinedOutput() + _, ok := err.(*exec.ExitError) + if !ok { + return nil, trace.TraceError(err) + } + lines = strings.Split(string(out), "\n") + return lines, nil +} + +func listProcessLinuxMac(text string) (lines []string, err error) { + cmd := exec.Command("ps", "aux") + out, err := cmd.CombinedOutput() + _, ok := err.(*exec.ExitError) + if !ok { + return nil, trace.TraceError(err) + } + _lines := strings.Split(string(out), "\n") + for _, l := range _lines { + if strings.Contains(l, text) { + lines = append(lines, l) + } + } + return lines, nil +} diff --git a/core/process/options.go b/core/process/options.go new file mode 100644 index 00000000..f0e046c9 --- /dev/null +++ b/core/process/options.go @@ -0,0 +1,20 @@ +package process + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type DaemonOption func(d interfaces.ProcessDaemon) + +func WithDaemonMaxErrors(maxErrors int) DaemonOption { + return func(d interfaces.ProcessDaemon) { + d.SetMaxErrors(maxErrors) + } +} + +func WithExitTimeout(timeout time.Duration) DaemonOption { + return func(d interfaces.ProcessDaemon) { + + } +} diff --git a/core/result/options.go b/core/result/options.go new file mode 100644 index 00000000..59c63301 --- /dev/null +++ b/core/result/options.go @@ -0,0 +1,16 @@ +package result + +import "go.mongodb.org/mongo-driver/bson/primitive" + +type Option func(opts *Options) + +type Options struct { + registryKey string // registry key + SpiderId primitive.ObjectID // data source id +} + +func WithRegistryKey(key string) Option { + return func(opts *Options) { + opts.registryKey = key + } +} diff --git a/core/result/service.go b/core/result/service.go new file mode 100644 index 00000000..3acd6321 --- /dev/null +++ b/core/result/service.go @@ -0,0 +1,89 @@ +package result + +import ( + "fmt" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" +) + +func NewResultService(registryKey string, s *models.Spider) (svc2 interfaces.ResultService, err error) { + // result service function + var fn interfaces.ResultServiceRegistryFn + + if registryKey == "" { + // default + fn = NewResultServiceMongo + } else { + // from registry + reg := GetResultServiceRegistry() + fn = reg.Get(registryKey) + if fn == nil { + return nil, errors.NewResultError(fmt.Sprintf("%s is not implemented", registryKey)) + } + } + + // generate result service + svc, err := fn(s.ColId, s.DataSourceId) + if err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} + +var store = sync.Map{} + +func GetResultService(spiderId primitive.ObjectID, opts ...Option) (svc2 interfaces.ResultService, err error) { + // model service + modelSvc, err := service.GetService() + if err != nil { + return nil, trace.TraceError(err) + } + + // spider + s, err := modelSvc.GetSpiderById(spiderId) + if err != nil { + return nil, trace.TraceError(err) + } + + // apply options + _opts := &Options{} + for _, opt := range opts { + opt(_opts) + } + + // store key + storeKey := s.ColId.Hex() + ":" + s.DataSourceId.Hex() + + // attempt to load result service from store + res, _ := store.Load(storeKey) + if res != nil { + svc, ok := res.(interfaces.ResultService) + if ok { + return svc, nil + } + } + + // registry key + var registryKey string + ds, _ := modelSvc.GetDataSourceById(s.DataSourceId) + if ds != nil { + registryKey = ds.Type + } + + // create a new result service if not exists + svc, err := NewResultService(registryKey, s) + if err != nil { + return nil, err + } + + // save into store + store.Store(storeKey, svc) + + return svc, nil +} diff --git a/core/result/service_mongo.go b/core/result/service_mongo.go new file mode 100644 index 00000000..cdc6ff3f --- /dev/null +++ b/core/result/service_mongo.go @@ -0,0 +1,146 @@ +package result + +import ( + "time" + + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +type ServiceMongo struct { + // dependencies + modelSvc service.ModelService + modelColSvc interfaces.ModelBaseService + + // internals + colId primitive.ObjectID // _id of models.DataCollection + dc *models.DataCollection // models.DataCollection + t time.Time +} + +func (svc *ServiceMongo) List(query generic.ListQuery, opts *generic.ListOptions) (results []interface{}, err error) { + _query := svc.getQuery(query) + _opts := svc.getOpts(opts) + return svc.getList(_query, _opts) +} + +func (svc *ServiceMongo) Count(query generic.ListQuery) (n int, err error) { + _query := svc.getQuery(query) + return svc.modelColSvc.Count(_query) +} + +func (svc *ServiceMongo) Insert(docs ...interface{}) (err error) { + if svc.dc.Dedup.Enabled && len(svc.dc.Dedup.Keys) > 0 { + for _, doc := range docs { + hash, err := utils.GetResultHash(doc, svc.dc.Dedup.Keys) + if err != nil { + return err + } + doc.(interfaces.Result).SetValue(constants.HashKey, hash) + query := bson.M{constants.HashKey: hash} + switch svc.dc.Dedup.Type { + case constants.DedupTypeOverwrite: + err = mongo.GetMongoCol(svc.dc.Name).ReplaceWithOptions(query, doc, &options.ReplaceOptions{Upsert: &[]bool{true}[0]}) + if err != nil { + return trace.TraceError(err) + } + default: + var o bson.M + err := mongo.GetMongoCol(svc.dc.Name).Find(query, &mongo.FindOptions{Limit: 1}).One(&o) + if err == nil { + // exists, ignore + continue + } + if err != mongo2.ErrNoDocuments { + // error + return trace.TraceError(err) + } + // not exists, insert + _, err = mongo.GetMongoCol(svc.dc.Name).Insert(doc) + if err != nil { + return trace.TraceError(err) + } + } + } + } else { + _, err = mongo.GetMongoCol(svc.dc.Name).InsertMany(docs) + if err != nil { + return trace.TraceError(err) + } + } + return nil +} + +func (svc *ServiceMongo) Index(fields []string) { + for _, field := range fields { + _ = mongo.GetMongoCol(svc.dc.Name).CreateIndex(mongo2.IndexModel{Keys: bson.M{field: 1}}) + } +} + +func (svc *ServiceMongo) SetTime(t time.Time) { + svc.t = t +} + +func (svc *ServiceMongo) GetTime() (t time.Time) { + return svc.t +} + +func (svc *ServiceMongo) getList(query bson.M, opts *mongo.FindOptions) (results []interface{}, err error) { + list, err := svc.modelColSvc.GetList(query, opts) + if err != nil { + return nil, err + } + for _, d := range list.GetModels() { + r, ok := d.(interfaces.Result) + if ok { + results = append(results, r) + } + } + return results, nil +} + +func (svc *ServiceMongo) getQuery(query generic.ListQuery) (res bson.M) { + return utils.GetMongoQuery(query) +} + +func (svc *ServiceMongo) getOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { + return utils.GetMongoOpts(opts) +} + +func NewResultServiceMongo(colId primitive.ObjectID, _ primitive.ObjectID) (svc2 interfaces.ResultService, err error) { + // service + svc := &ServiceMongo{ + colId: colId, + t: time.Now(), + } + + // dependency injection + svc.modelSvc, err = service.GetService() + if err != nil { + return nil, err + } + + // data collection + svc.dc, _ = svc.modelSvc.GetDataCollectionById(colId) + go func() { + for { + time.Sleep(1 * time.Second) + svc.dc, _ = svc.modelSvc.GetDataCollectionById(colId) + } + }() + + // data collection model service + svc.modelColSvc = service.GetBaseServiceByColName(interfaces.ModelIdResult, svc.dc.Name) + + return svc, nil +} diff --git a/core/result/service_registry.go b/core/result/service_registry.go new file mode 100644 index 00000000..2c635397 --- /dev/null +++ b/core/result/service_registry.go @@ -0,0 +1,48 @@ +package result + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "sync" +) + +type ServiceRegistry struct { + // internals + services sync.Map +} + +func (r *ServiceRegistry) Register(key string, fn interfaces.ResultServiceRegistryFn) { + r.services.Store(key, fn) +} + +func (r *ServiceRegistry) Unregister(key string) { + r.services.Delete(key) +} + +func (r *ServiceRegistry) Get(key string) (fn interfaces.ResultServiceRegistryFn) { + res, ok := r.services.Load(key) + if ok { + fn, ok = res.(interfaces.ResultServiceRegistryFn) + if !ok { + return nil + } + return fn + } + return nil +} + +func NewResultServiceRegistry() (r interfaces.ResultServiceRegistry) { + r = &ServiceRegistry{ + services: sync.Map{}, + } + return r +} + +var _svc interfaces.ResultServiceRegistry + +func GetResultServiceRegistry() (r interfaces.ResultServiceRegistry) { + if _svc != nil { + return _svc + } + _svc = NewResultServiceRegistry() + return _svc +} diff --git a/core/result/test/base.go b/core/result/test/base.go new file mode 100644 index 00000000..14331b20 --- /dev/null +++ b/core/result/test/base.go @@ -0,0 +1,76 @@ +package test + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "go.uber.org/dig" + "testing" +) + +func init() { + T = NewTest() +} + +var T *Test + +type Test struct { + // dependencies + modelSvc service.ModelService + resultSvc interfaces.ResultService + + // test data + TestColName string + TestCol *mongo.Col + TestDc *models.DataCollection +} + +func (t *Test) Setup(t2 *testing.T) { + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = t.modelSvc.DropAll() +} + +func NewTest() *Test { + var err error + + // test + t := &Test{ + TestColName: "test_results", + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.NewService); err != nil { + panic(err) + } + if err := c.Invoke(func( + modelSvc service.ModelService, + ) { + t.modelSvc = modelSvc + }); err != nil { + panic(err) + } + + // data collection + t.TestDc = &models.DataCollection{ + Name: t.TestColName, + } + if err := delegate.NewModelDelegate(t.TestDc).Add(); err != nil { + panic(err) + } + t.TestCol = mongo.GetMongoCol(t.TestColName) + + // result service + t.resultSvc, err = result.GetResultService(t.TestDc.GetId()) + if err != nil { + panic(err) + } + + return t +} diff --git a/core/result/test/service_test.go b/core/result/test/service_test.go new file mode 100644 index 00000000..b44faf4e --- /dev/null +++ b/core/result/test/service_test.go @@ -0,0 +1,67 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/stretchr/testify/require" + "testing" +) + +func TestResultService_GetList(t *testing.T) { + var err error + T.Setup(t) + + n := 1000 + var docs []interface{} + for i := 0; i < n; i++ { + d := &models.Result{ + "i": i, + } + docs = append(docs, d) + } + _, err = T.TestCol.InsertMany(docs) + require.Nil(t, err) + + // get all + results, err := T.resultSvc.List(nil, nil) + require.Nil(t, err) + require.Equal(t, n, len(results)) + + //query := bson.M{ + // "i": bson.M{ + // "$lt": n / 2, + // }, + //} + //results, err = T.resultSvc.List(query, nil) + //require.Nil(t, err) + //require.Equal(t, n/2, len(results)) +} + +func TestResultService_Count(t *testing.T) { + var err error + T.Setup(t) + + n := 1000 + var docs []interface{} + for i := 0; i < n; i++ { + d := &models.Result{ + "i": i, + } + docs = append(docs, d) + } + _, err = T.TestCol.InsertMany(docs) + require.Nil(t, err) + + // get all + total, err := T.resultSvc.Count(nil) + require.Nil(t, err) + require.Equal(t, n, total) + + //query := bson.M{ + // "i": bson.M{ + // "$lt": n / 2, + // }, + //} + //total, err = T.resultSvc.Count(query) + //require.Nil(t, err) + //require.Equal(t, n/2, total) +} diff --git a/core/routes/group.go b/core/routes/group.go new file mode 100644 index 00000000..cade1af4 --- /dev/null +++ b/core/routes/group.go @@ -0,0 +1,20 @@ +package routes + +import ( + "github.com/crawlab-team/crawlab/core/middlewares" + "github.com/gin-gonic/gin" +) + +type RouterGroups struct { + AuthGroup *gin.RouterGroup + AnonymousGroup *gin.RouterGroup + FilerGroup *gin.RouterGroup +} + +func NewRouterGroups(app *gin.Engine) (groups *RouterGroups) { + return &RouterGroups{ + AuthGroup: app.Group("/", middlewares.AuthorizationMiddleware()), + AnonymousGroup: app.Group("/"), + FilerGroup: app.Group("/filer", middlewares.FilerAuthorizationMiddleware()), + } +} diff --git a/core/routes/router.go b/core/routes/router.go new file mode 100644 index 00000000..c82603bc --- /dev/null +++ b/core/routes/router.go @@ -0,0 +1,178 @@ +package routes + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/controllers" + "github.com/gin-gonic/gin" + "net/http" + "path" +) + +type RouterServiceInterface interface { + RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) + RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) +} + +type RouterService struct { + app *gin.Engine +} + +func NewRouterService(app *gin.Engine) (svc *RouterService) { + return &RouterService{ + app: app, + } +} + +func (svc *RouterService) RegisterControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.BasicController) { + group.GET(basePath, ctr.Get) + group.POST(basePath, ctr.Post) + group.PUT(basePath, ctr.Put) + group.DELETE(basePath, ctr.Delete) +} + +func (svc *RouterService) RegisterListControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListController) { + group.GET(basePath+"/:id", ctr.Get) + group.GET(basePath, ctr.GetList) + group.POST(basePath, ctr.Post) + group.POST(basePath+"/batch", ctr.PostList) + group.PUT(basePath+"/:id", ctr.Put) + group.PUT(basePath, ctr.PutList) + group.DELETE(basePath+"/:id", ctr.Delete) + group.DELETE(basePath, ctr.DeleteList) +} + +func (svc *RouterService) RegisterActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ActionController) { + for _, action := range ctr.Actions() { + routerPath := path.Join(basePath, action.Path) + switch action.Method { + case http.MethodGet: + group.GET(routerPath, action.HandlerFunc) + case http.MethodPost: + group.POST(routerPath, action.HandlerFunc) + case http.MethodPut: + group.PUT(routerPath, action.HandlerFunc) + case http.MethodDelete: + group.DELETE(routerPath, action.HandlerFunc) + } + } +} + +func (svc *RouterService) RegisterListActionControllerToGroup(group *gin.RouterGroup, basePath string, ctr controllers.ListActionController) { + svc.RegisterListControllerToGroup(group, basePath, ctr) + svc.RegisterActionControllerToGroup(group, basePath, ctr) +} + +func (svc *RouterService) RegisterHandlerToGroup(group *gin.RouterGroup, path string, method string, handler gin.HandlerFunc) { + switch method { + case http.MethodGet: + group.GET(path, handler) + case http.MethodPost: + group.POST(path, handler) + case http.MethodPut: + group.PUT(path, handler) + case http.MethodDelete: + group.DELETE(path, handler) + default: + log.Warn(fmt.Sprintf("%s is not a valid http method", method)) + } +} + +func InitRoutes(app *gin.Engine) (err error) { + // routes groups + groups := NewRouterGroups(app) + + // router service + svc := NewRouterService(app) + + // register routes + registerRoutesAnonymousGroup(svc, groups) + registerRoutesAuthGroup(svc, groups) + registerRoutesFilterGroup(svc, groups) + + return nil +} + +func registerRoutesAnonymousGroup(svc *RouterService, groups *RouterGroups) { + // login + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/", controllers.LoginController) + + // version + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/version", controllers.VersionController) + + // system info + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/system-info", controllers.SystemInfoController) + + // demo + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/demo", controllers.DemoController) + + // sync + svc.RegisterActionControllerToGroup(groups.AnonymousGroup, "/sync", controllers.SyncController) +} + +func registerRoutesAuthGroup(svc *RouterService, groups *RouterGroups) { + // node + svc.RegisterListControllerToGroup(groups.AuthGroup, "/nodes", controllers.NodeController) + + // project + svc.RegisterListControllerToGroup(groups.AuthGroup, "/projects", controllers.ProjectController) + + // user + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/users", controllers.UserController) + + // spider + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/spiders", controllers.SpiderController) + + // task + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/tasks", controllers.TaskController) + + // tag + svc.RegisterListControllerToGroup(groups.AuthGroup, "/tags", controllers.TagController) + + // setting + svc.RegisterListControllerToGroup(groups.AuthGroup, "/settings", controllers.SettingController) + + // data collection + svc.RegisterListControllerToGroup(groups.AuthGroup, "/data/collections", controllers.DataCollectionController) + + // result + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/results", controllers.ResultController) + + // schedule + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/schedules", controllers.ScheduleController) + + // stats + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/stats", controllers.StatsController) + + // token + svc.RegisterListControllerToGroup(groups.AuthGroup, "/tokens", controllers.TokenController) + + // git + svc.RegisterListControllerToGroup(groups.AuthGroup, "/gits", controllers.GitController) + + // role + svc.RegisterListControllerToGroup(groups.AuthGroup, "/roles", controllers.RoleController) + + // permission + svc.RegisterListControllerToGroup(groups.AuthGroup, "/permissions", controllers.PermissionController) + + // export + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/export", controllers.ExportController) + + // notification + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/notifications", controllers.NotificationController) + + // filter + svc.RegisterActionControllerToGroup(groups.AuthGroup, "/filters", controllers.FilterController) + + // data sources + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/data-sources", controllers.DataSourceController) + + // environments + svc.RegisterListActionControllerToGroup(groups.AuthGroup, "/environments", controllers.EnvironmentController) +} + +func registerRoutesFilterGroup(svc *RouterService, groups *RouterGroups) { + // filer + svc.RegisterActionControllerToGroup(groups.FilerGroup, "", controllers.FilerController) +} diff --git a/core/routes/router_test.go b/core/routes/router_test.go new file mode 100644 index 00000000..7b72c852 --- /dev/null +++ b/core/routes/router_test.go @@ -0,0 +1,26 @@ +package routes + +import ( + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/require" + "net/http" + "testing" + "time" +) + +func TestInitRoutes(t *testing.T) { + app := gin.New() + err := InitRoutes(app) + require.Nil(t, err) + + srv := &http.Server{ + Handler: app, + Addr: "localhost:8000", + } + go func() { + err = srv.ListenAndServe() + require.Nil(t, err) + }() + + time.Sleep(5 * time.Second) +} diff --git a/core/schedule/logger.go b/core/schedule/logger.go new file mode 100644 index 00000000..fa6bf834 --- /dev/null +++ b/core/schedule/logger.go @@ -0,0 +1,35 @@ +package schedule + +import ( + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/go-trace" + "github.com/robfig/cron/v3" + "strings" +) + +type Logger struct { +} + +func (l *Logger) Info(msg string, keysAndValues ...interface{}) { + p := l.getPlaceholder(len(keysAndValues)) + log.Infof(fmt.Sprintf("cron: %s %s", msg, p), keysAndValues...) +} + +func (l *Logger) Error(err error, msg string, keysAndValues ...interface{}) { + p := l.getPlaceholder(len(keysAndValues)) + log.Errorf(fmt.Sprintf("cron: %s %s", msg, p), keysAndValues...) + trace.PrintError(err) +} + +func (l *Logger) getPlaceholder(n int) (s string) { + var arr []string + for i := 0; i < n; i++ { + arr = append(arr, "%v") + } + return strings.Join(arr, " ") +} + +func NewLogger() cron.Logger { + return &Logger{} +} diff --git a/core/schedule/options.go b/core/schedule/options.go new file mode 100644 index 00000000..6ad22f99 --- /dev/null +++ b/core/schedule/options.go @@ -0,0 +1,37 @@ +package schedule + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.ScheduleService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.ScheduleService) { + svc.SetConfigPath(path) + } +} + +func WithLocation(loc *time.Location) Option { + return func(svc interfaces.ScheduleService) { + svc.SetLocation(loc) + } +} + +func WithDelayIfStillRunning() Option { + return func(svc interfaces.ScheduleService) { + svc.SetDelay(true) + } +} + +func WithSkipIfStillRunning() Option { + return func(svc interfaces.ScheduleService) { + svc.SetSkip(true) + } +} + +func WithUpdateInterval(interval time.Duration) Option { + return func(svc interfaces.ScheduleService) { + } +} diff --git a/core/schedule/service.go b/core/schedule/service.go new file mode 100644 index 00000000..f4b978da --- /dev/null +++ b/core/schedule/service.go @@ -0,0 +1,287 @@ +package schedule + +import ( + "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type Service struct { + // dependencies + interfaces.WithConfigPath + modelSvc service.ModelService + adminSvc interfaces.SpiderAdminService + + // settings variables + loc *time.Location + delay bool + skip bool + updateInterval time.Duration + + // internals + cron *cron.Cron + logger cron.Logger + schedules []models.Schedule + stopped bool + mu sync.Mutex +} + +func (svc *Service) GetLocation() (loc *time.Location) { + return svc.loc +} + +func (svc *Service) SetLocation(loc *time.Location) { + svc.loc = loc +} + +func (svc *Service) GetDelay() (delay bool) { + return svc.delay +} + +func (svc *Service) SetDelay(delay bool) { + svc.delay = delay +} + +func (svc *Service) GetSkip() (skip bool) { + return svc.skip +} + +func (svc *Service) SetSkip(skip bool) { + svc.skip = skip +} + +func (svc *Service) GetUpdateInterval() (interval time.Duration) { + return svc.updateInterval +} + +func (svc *Service) SetUpdateInterval(interval time.Duration) { + svc.updateInterval = interval +} + +func (svc *Service) Init() (err error) { + return svc.fetch() +} + +func (svc *Service) Start() { + svc.cron.Start() + go svc.Update() +} + +func (svc *Service) Wait() { + utils.DefaultWait() + svc.Stop() +} + +func (svc *Service) Stop() { + svc.stopped = true + svc.cron.Stop() +} + +func (svc *Service) Enable(s interfaces.Schedule, args ...interface{}) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + id, err := svc.cron.AddFunc(s.GetCron(), svc.schedule(s.GetId())) + if err != nil { + return trace.TraceError(err) + } + s.SetEnabled(true) + s.SetEntryId(id) + u := utils.GetUserFromArgs(args...) + return delegate.NewModelDelegate(s, u).Save() +} + +func (svc *Service) Disable(s interfaces.Schedule, args ...interface{}) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + svc.cron.Remove(s.GetEntryId()) + s.SetEnabled(false) + s.SetEntryId(-1) + u := utils.GetUserFromArgs(args...) + return delegate.NewModelDelegate(s, u).Save() +} + +func (svc *Service) Update() { + for { + if svc.stopped { + return + } + + svc.update() + + time.Sleep(svc.updateInterval) + } +} + +func (svc *Service) GetCron() (c *cron.Cron) { + return svc.cron +} + +func (svc *Service) update() { + // fetch enabled schedules + if err := svc.fetch(); err != nil { + trace.PrintError(err) + return + } + + // entry id map + entryIdsMap := svc.getEntryIdsMap() + + // iterate enabled schedules + for _, s := range svc.schedules { + _, ok := entryIdsMap[s.EntryId] + if ok { + entryIdsMap[s.EntryId] = true + } else { + if err := svc.Enable(&s); err != nil { + trace.PrintError(err) + continue + } + } + } + + // remove non-existent entries + for id, ok := range entryIdsMap { + if !ok { + svc.cron.Remove(id) + } + } +} + +func (svc *Service) getEntryIdsMap() (res map[cron.EntryID]bool) { + res = map[cron.EntryID]bool{} + for _, e := range svc.cron.Entries() { + res[e.ID] = false + } + return res +} + +func (svc *Service) fetch() (err error) { + query := bson.M{ + "enabled": true, + } + svc.schedules, err = svc.modelSvc.GetScheduleList(query, nil) + if err != nil { + return err + } + return nil +} + +func (svc *Service) schedule(id primitive.ObjectID) (fn func()) { + return func() { + // schedule + s, err := svc.modelSvc.GetScheduleById(id) + if err != nil { + trace.PrintError(err) + return + } + + // spider + spider, err := svc.modelSvc.GetSpiderById(s.GetSpiderId()) + if err != nil { + trace.PrintError(err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: s.GetMode(), + NodeIds: s.GetNodeIds(), + Cmd: s.GetCmd(), + Param: s.GetParam(), + Priority: s.GetPriority(), + ScheduleId: s.GetId(), + UserId: s.UserId, + } + + // normalize options + if opts.Mode == "" { + opts.Mode = spider.Mode + } + if len(opts.NodeIds) == 0 { + opts.NodeIds = spider.NodeIds + } + if opts.Cmd == "" { + opts.Cmd = spider.Cmd + } + if opts.Param == "" { + opts.Param = spider.Param + } + if opts.Priority == 0 { + if spider.Priority > 0 { + opts.Priority = spider.Priority + } else { + opts.Priority = 5 + } + } + + // schedule or assign a task in the task queue + if _, err := svc.adminSvc.Schedule(s.GetSpiderId(), opts); err != nil { + trace.PrintError(err) + } + } +} + +func NewScheduleService() (svc2 interfaces.ScheduleService, err error) { + // service + svc := &Service{ + WithConfigPath: config.NewConfigPathService(), + loc: time.Local, + // TODO: implement delay and skip + delay: false, + skip: false, + updateInterval: 1 * time.Minute, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + modelSvc service.ModelService, + adminSvc interfaces.SpiderAdminService, + ) { + svc.modelSvc = modelSvc + svc.adminSvc = adminSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // logger + svc.logger = NewLogger() + + // cron + svc.cron = cron.New( + cron.WithLogger(svc.logger), + cron.WithLocation(svc.loc), + cron.WithChain(cron.Recover(svc.logger)), + ) + + // initialize + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +var svc interfaces.ScheduleService + +func GetScheduleService() (res interfaces.ScheduleService, err error) { + if svc != nil { + return svc, nil + } + svc, err = NewScheduleService() + if err != nil { + return nil, err + } + return svc, nil +} diff --git a/core/schedule/service_v2.go b/core/schedule/service_v2.go new file mode 100644 index 00000000..c1cffbef --- /dev/null +++ b/core/schedule/service_v2.go @@ -0,0 +1,283 @@ +package schedule + +import ( + "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/spider/admin" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/robfig/cron/v3" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + interfaces.WithConfigPath + modelSvc *service.ModelServiceV2[models.ScheduleV2] + adminSvc *admin.ServiceV2 + + // settings variables + loc *time.Location + delay bool + skip bool + updateInterval time.Duration + + // internals + cron *cron.Cron + logger cron.Logger + schedules []models.ScheduleV2 + stopped bool + mu sync.Mutex +} + +func (svc *ServiceV2) GetLocation() (loc *time.Location) { + return svc.loc +} + +func (svc *ServiceV2) SetLocation(loc *time.Location) { + svc.loc = loc +} + +func (svc *ServiceV2) GetDelay() (delay bool) { + return svc.delay +} + +func (svc *ServiceV2) SetDelay(delay bool) { + svc.delay = delay +} + +func (svc *ServiceV2) GetSkip() (skip bool) { + return svc.skip +} + +func (svc *ServiceV2) SetSkip(skip bool) { + svc.skip = skip +} + +func (svc *ServiceV2) GetUpdateInterval() (interval time.Duration) { + return svc.updateInterval +} + +func (svc *ServiceV2) SetUpdateInterval(interval time.Duration) { + svc.updateInterval = interval +} + +func (svc *ServiceV2) Init() (err error) { + return svc.fetch() +} + +func (svc *ServiceV2) Start() { + svc.cron.Start() + go svc.Update() +} + +func (svc *ServiceV2) Wait() { + utils.DefaultWait() + svc.Stop() +} + +func (svc *ServiceV2) Stop() { + svc.stopped = true + svc.cron.Stop() +} + +func (svc *ServiceV2) Enable(s models.ScheduleV2, by primitive.ObjectID) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + id, err := svc.cron.AddFunc(s.Cron, svc.schedule(s.Id)) + if err != nil { + return trace.TraceError(err) + } + s.Enabled = true + s.EntryId = id + s.SetUpdated(by) + return svc.modelSvc.ReplaceById(s.Id, s) +} + +func (svc *ServiceV2) Disable(s models.ScheduleV2, by primitive.ObjectID) (err error) { + svc.mu.Lock() + defer svc.mu.Unlock() + + svc.cron.Remove(s.EntryId) + s.Enabled = false + s.EntryId = -1 + s.SetUpdated(by) + return svc.modelSvc.ReplaceById(s.Id, s) +} + +func (svc *ServiceV2) Update() { + for { + if svc.stopped { + return + } + + svc.update() + + time.Sleep(svc.updateInterval) + } +} + +func (svc *ServiceV2) GetCron() (c *cron.Cron) { + return svc.cron +} + +func (svc *ServiceV2) update() { + // fetch enabled schedules + if err := svc.fetch(); err != nil { + trace.PrintError(err) + return + } + + // entry id map + entryIdsMap := svc.getEntryIdsMap() + + // iterate enabled schedules + for _, s := range svc.schedules { + _, ok := entryIdsMap[s.EntryId] + if ok { + entryIdsMap[s.EntryId] = true + } else { + if !s.Enabled { + err := svc.Enable(s, s.GetCreatedBy()) + if err != nil { + trace.PrintError(err) + continue + } + } + } + } + + // remove non-existent entries + for id, ok := range entryIdsMap { + if !ok { + svc.cron.Remove(id) + } + } +} + +func (svc *ServiceV2) getEntryIdsMap() (res map[cron.EntryID]bool) { + res = map[cron.EntryID]bool{} + for _, e := range svc.cron.Entries() { + res[e.ID] = false + } + return res +} + +func (svc *ServiceV2) fetch() (err error) { + query := bson.M{ + "enabled": true, + } + svc.schedules, err = svc.modelSvc.GetMany(query, nil) + if err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) schedule(id primitive.ObjectID) (fn func()) { + return func() { + // schedule + s, err := svc.modelSvc.GetById(id) + if err != nil { + trace.PrintError(err) + return + } + + // spider + spider, err := service.NewModelServiceV2[models.SpiderV2]().GetById(s.SpiderId) + if err != nil { + trace.PrintError(err) + return + } + + // options + opts := &interfaces.SpiderRunOptions{ + Mode: s.Mode, + NodeIds: s.NodeIds, + Cmd: s.Cmd, + Param: s.Param, + Priority: s.Priority, + ScheduleId: s.Id, + UserId: s.GetCreatedBy(), + } + + // normalize options + if opts.Mode == "" { + opts.Mode = spider.Mode + } + if len(opts.NodeIds) == 0 { + opts.NodeIds = spider.NodeIds + } + if opts.Cmd == "" { + opts.Cmd = spider.Cmd + } + if opts.Param == "" { + opts.Param = spider.Param + } + if opts.Priority == 0 { + if spider.Priority > 0 { + opts.Priority = spider.Priority + } else { + opts.Priority = 5 + } + } + + // schedule or assign a task in the task queue + if _, err := svc.adminSvc.Schedule(s.SpiderId, opts); err != nil { + trace.PrintError(err) + } + } +} + +func NewScheduleServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + WithConfigPath: config.NewConfigPathService(), + loc: time.Local, + // TODO: implement delay and skip + delay: false, + skip: false, + updateInterval: 1 * time.Minute, + } + svc.adminSvc, err = admin.GetSpiderAdminServiceV2() + if err != nil { + return nil, err + } + svc.modelSvc = service.NewModelServiceV2[models.ScheduleV2]() + + // logger + svc.logger = NewLogger() + + // cron + svc.cron = cron.New( + cron.WithLogger(svc.logger), + cron.WithLocation(svc.loc), + cron.WithChain(cron.Recover(svc.logger)), + ) + + // initialize + if err := svc.Init(); err != nil { + return nil, err + } + + return svc, nil +} + +var svcV2 *ServiceV2 + +func GetScheduleServiceV2() (res *ServiceV2, err error) { + if svcV2 != nil { + return svcV2, nil + } + svcV2, err = NewScheduleServiceV2() + if err != nil { + return nil, err + } + return svcV2, nil +} diff --git a/core/schedule/test/base.go b/core/schedule/test/base.go new file mode 100644 index 00000000..cd7c885e --- /dev/null +++ b/core/schedule/test/base.go @@ -0,0 +1,91 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/schedule" + "go.uber.org/dig" + "testing" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + // dependencies + modelSvc service.ModelService + scheduleSvc interfaces.ScheduleService + + // test data + TestSchedule interfaces.Schedule + TestSpider interfaces.Spider + ScriptName string + Script string +} + +func (t *Test) Setup(t2 *testing.T) { + t.scheduleSvc.Start() + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + t.scheduleSvc.Stop() + _ = t.modelSvc.GetBaseService(interfaces.ModelIdTask).Delete(nil) +} + +func NewTest() (t *Test, err error) { + // test + t = &Test{ + TestSpider: &models.Spider{ + Name: "test_spider", + Cmd: "go run main.go", + }, + ScriptName: "main.go", + Script: `package main +import "fmt" +func main() { + fmt.Println("it works") +}`, + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.GetService); err != nil { + return nil, err + } + if err := c.Provide(schedule.NewScheduleService); err != nil { + return nil, err + } + if err := c.Invoke(func(modelSvc service.ModelService, scheduleSvc interfaces.ScheduleService) { + t.modelSvc = modelSvc + t.scheduleSvc = scheduleSvc + }); err != nil { + return nil, err + } + + // add spider to db + if err := delegate.NewModelDelegate(t.TestSpider).Add(); err != nil { + return nil, err + } + + // test schedule + t.TestSchedule = &models.Schedule{ + Name: "test_schedule", + SpiderId: t.TestSpider.GetId(), + Cron: "* * * * *", + } + if err := delegate.NewModelDelegate(t.TestSchedule).Add(); err != nil { + return nil, err + } + + return t, nil +} diff --git a/core/schedule/test/schedule_service_test.go b/core/schedule/test/schedule_service_test.go new file mode 100644 index 00000000..7a815da1 --- /dev/null +++ b/core/schedule/test/schedule_service_test.go @@ -0,0 +1,44 @@ +package test + +import ( + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestScheduleService_Enable_Disable(t *testing.T) { + var err error + T.Setup(t) + + time.Sleep(1 * time.Second) + err = T.scheduleSvc.Enable(T.TestSchedule) + require.Nil(t, err) + time.Sleep(1 * time.Second) + + require.True(t, T.TestSchedule.GetEnabled()) + require.Greater(t, int(T.TestSchedule.GetEntryId()), -1) + e := T.scheduleSvc.GetCron().Entry(T.TestSchedule.GetEntryId()) + require.Equal(t, T.TestSchedule.GetEntryId(), e.ID) + time.Sleep(1 * time.Second) + + err = T.scheduleSvc.Disable(T.TestSchedule) + require.False(t, T.TestSchedule.GetEnabled()) + require.Equal(t, 0, len(T.scheduleSvc.GetCron().Entries())) +} + +func TestScheduleService_Run(t *testing.T) { + var err error + T.Setup(t) + + time.Sleep(1 * time.Second) + err = T.scheduleSvc.Enable(T.TestSchedule) + require.Nil(t, err) + time.Sleep(1 * time.Minute) + + tasks, err := T.modelSvc.GetTaskList(nil, nil) + require.Nil(t, err) + require.Greater(t, len(tasks), 0) + for _, task := range tasks { + require.False(t, task.ScheduleId.IsZero()) + } +} diff --git a/core/spider/admin/options.go b/core/spider/admin/options.go new file mode 100644 index 00000000..32e08e76 --- /dev/null +++ b/core/spider/admin/options.go @@ -0,0 +1,11 @@ +package admin + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(svc interfaces.SpiderAdminService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.SpiderAdminService) { + svc.SetConfigPath(path) + } +} diff --git a/core/spider/admin/service.go b/core/spider/admin/service.go new file mode 100644 index 00000000..83d6ac9e --- /dev/null +++ b/core/spider/admin/service.go @@ -0,0 +1,352 @@ +package admin + +import ( + "context" + "github.com/apex/log" + vcs "github.com/crawlab-team/crawlab-vcs" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/google/uuid" + "github.com/robfig/cron/v3" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "path" + "path/filepath" + "sync" + "time" +) + +type Service struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + schedulerSvc interfaces.TaskSchedulerService + cron *cron.Cron + syncLock bool + + // settings + cfgPath string +} + +func (svc *Service) GetConfigPath() (path string) { + return svc.cfgPath +} + +func (svc *Service) SetConfigPath(path string) { + svc.cfgPath = path +} + +func (svc *Service) Start() (err error) { + return svc.SyncGit() +} + +func (svc *Service) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // spider + s, err := svc.modelSvc.GetSpiderById(id) + if err != nil { + return nil, err + } + + // assign tasks + return svc.scheduleTasks(s, opts) +} + +func (svc *Service) Clone(id primitive.ObjectID, opts *interfaces.SpiderCloneOptions) (err error) { + // TODO: implement + return nil +} + +func (svc *Service) Delete(id primitive.ObjectID) (err error) { + panic("implement me") +} + +func (svc *Service) SyncGit() (err error) { + if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { + return trace.TraceError(err) + } + svc.cron.Start() + return nil +} + +func (svc *Service) SyncGitOne(g interfaces.Git) (err error) { + svc.syncGitOne(g) + return nil +} + +func (svc *Service) Export(id primitive.ObjectID) (filePath string, err error) { + // spider fs + workspacePath := viper.GetString("workspace") + spiderFolderPath := filepath.Join(workspacePath, id.Hex()) + + // zip files in workspace + dirPath := spiderFolderPath + zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + +func (svc *Service) scheduleTasks(s *models.Spider, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // main task + mainTask := &models.Task{ + SpiderId: s.Id, + Mode: opts.Mode, + NodeIds: opts.NodeIds, + Cmd: opts.Cmd, + Param: opts.Param, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + + // normalize + if mainTask.Mode == "" { + mainTask.Mode = s.Mode + } + if mainTask.NodeIds == nil { + mainTask.NodeIds = s.NodeIds + } + if mainTask.Cmd == "" { + mainTask.Cmd = s.Cmd + } + if mainTask.Param == "" { + mainTask.Param = s.Param + } + if mainTask.Priority == 0 { + mainTask.Priority = s.Priority + } + + if svc.isMultiTask(opts) { + // multi tasks + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + for _, nodeId := range nodeIds { + t := &models.Task{ + SpiderId: s.Id, + Mode: opts.Mode, + Cmd: opts.Cmd, + Param: opts.Param, + NodeId: nodeId, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + t2, err := svc.schedulerSvc.Enqueue(t) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.GetId()) + } + } else { + // single task + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + if len(nodeIds) > 0 { + mainTask.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(mainTask) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.GetId()) + } + + return taskIds, nil +} + +func (svc *Service) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []primitive.ObjectID, err error) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := svc.modelSvc.GetNodeList(query, nil) + if err != nil { + return nil, err + } + for _, node := range nodes { + nodeIds = append(nodeIds, node.GetId()) + } + } else if opts.Mode == constants.RunTypeSelectedNodes { + nodeIds = opts.NodeIds + } + return nodeIds, nil +} + +func (svc *Service) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := svc.modelSvc.GetNodeList(query, nil) + if err != nil { + trace.PrintError(err) + return false + } + return len(nodes) > 1 + } else if opts.Mode == constants.RunTypeRandom { + return false + } else if opts.Mode == constants.RunTypeSelectedNodes { + return len(opts.NodeIds) > 1 + } else { + return false + } +} + +func (svc *Service) syncGit() { + if svc.syncLock { + log.Infof("[SpiderAdminService] sync git is locked, skip") + return + } + log.Infof("[SpiderAdminService] start to sync git") + + svc.syncLock = true + defer func() { + svc.syncLock = false + }() + + // spiders + spiders, err := svc.modelSvc.GetSpiderList(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + + // spider ids + var spiderIds []primitive.ObjectID + for _, s := range spiders { + spiderIds = append(spiderIds, s.Id) + } + + if len(spiderIds) > 0 { + // gits + gits, err := svc.modelSvc.GetGitList(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + "auto_pull": true, + }, nil) + if err != nil { + trace.PrintError(err) + return + } + + wg := sync.WaitGroup{} + wg.Add(len(gits)) + for _, g := range gits { + go func(g models.Git) { + svc.syncGitOne(&g) + wg.Done() + }(g) + } + wg.Wait() + } + + log.Infof("[SpiderAdminService] finished sync git") +} + +func (svc *Service) syncGitOne(g interfaces.Git) { + log.Infof("[SpiderAdminService] sync git %s", g.GetId()) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // git client + workspacePath := viper.GetString("workspace") + gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.GetId().Hex()))) + if err != nil { + return + } + + // set auth + utils.InitGitClientAuth(g, gitClient) + + // check if remote has changes + ok, err := gitClient.IsRemoteChanged() + if err != nil { + trace.PrintError(err) + return + } + if !ok { + // no change + return + } + + // pull and sync to workspace + if err := gitClient.Reset(); err != nil { + trace.PrintError(err) + return + } + if err := gitClient.Pull(); err != nil { + trace.PrintError(err) + return + } + + // wait for context to end + <-ctx.Done() +} + +func NewSpiderAdminService(opts ...Option) (svc2 interfaces.SpiderAdminService, err error) { + svc := &Service{ + cfgPath: config2.GetConfigPath(), + } + + // apply options + for _, opt := range opts { + opt(svc) + } + + // dependency injection + if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService, schedulerSvc interfaces.TaskSchedulerService) { + svc.nodeCfgSvc = nodeCfgSvc + svc.modelSvc = modelSvc + svc.schedulerSvc = schedulerSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // cron + svc.cron = cron.New() + + // validate node type + if !svc.nodeCfgSvc.IsMaster() { + return nil, trace.TraceError(errors.ErrorSpiderForbidden) + } + + return svc, nil +} + +var _service interfaces.SpiderAdminService + +func GetSpiderAdminService() (svc2 interfaces.SpiderAdminService, err error) { + if _service != nil { + return _service, nil + } + + _service, err = NewSpiderAdminService() + if err != nil { + return nil, err + } + + return _service, nil +} diff --git a/core/spider/admin/service_v2.go b/core/spider/admin/service_v2.go new file mode 100644 index 00000000..73a84596 --- /dev/null +++ b/core/spider/admin/service_v2.go @@ -0,0 +1,328 @@ +package admin + +import ( + "context" + "github.com/apex/log" + vcs "github.com/crawlab-team/crawlab-vcs" + config2 "github.com/crawlab-team/crawlab/core/config" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/task/scheduler" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/google/uuid" + "github.com/robfig/cron/v3" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "path" + "path/filepath" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + schedulerSvc *scheduler.ServiceV2 + cron *cron.Cron + syncLock bool + + // settings + cfgPath string +} + +func (svc *ServiceV2) Start() (err error) { + return svc.SyncGit() +} + +func (svc *ServiceV2) Schedule(id primitive.ObjectID, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // spider + s, err := service.NewModelServiceV2[models.SpiderV2]().GetById(id) + if err != nil { + return nil, err + } + + // assign tasks + return svc.scheduleTasks(s, opts) +} + +func (svc *ServiceV2) SyncGit() (err error) { + if _, err = svc.cron.AddFunc("* * * * *", svc.syncGit); err != nil { + return trace.TraceError(err) + } + svc.cron.Start() + return nil +} + +func (svc *ServiceV2) SyncGitOne(g *models.GitV2) (err error) { + svc.syncGitOne(g) + return nil +} + +func (svc *ServiceV2) Export(id primitive.ObjectID) (filePath string, err error) { + // spider fs + workspacePath := viper.GetString("workspace") + spiderFolderPath := filepath.Join(workspacePath, id.Hex()) + + // zip files in workspace + dirPath := spiderFolderPath + zipFilePath := path.Join(os.TempDir(), uuid.New().String()+".zip") + if err := utils.ZipDirectory(dirPath, zipFilePath); err != nil { + return "", trace.TraceError(err) + } + + return zipFilePath, nil +} + +func (svc *ServiceV2) scheduleTasks(s *models.SpiderV2, opts *interfaces.SpiderRunOptions) (taskIds []primitive.ObjectID, err error) { + // main task + mainTask := &models.TaskV2{ + SpiderId: s.Id, + Mode: opts.Mode, + NodeIds: opts.NodeIds, + Cmd: opts.Cmd, + Param: opts.Param, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + mainTask.SetId(primitive.NewObjectID()) + + // normalize + if mainTask.Mode == "" { + mainTask.Mode = s.Mode + } + if mainTask.NodeIds == nil { + mainTask.NodeIds = s.NodeIds + } + if mainTask.Cmd == "" { + mainTask.Cmd = s.Cmd + } + if mainTask.Param == "" { + mainTask.Param = s.Param + } + if mainTask.Priority == 0 { + mainTask.Priority = s.Priority + } + + if svc.isMultiTask(opts) { + // multi tasks + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + for _, nodeId := range nodeIds { + t := &models.TaskV2{ + SpiderId: s.Id, + Mode: opts.Mode, + Cmd: opts.Cmd, + Param: opts.Param, + NodeId: nodeId, + ScheduleId: opts.ScheduleId, + Priority: opts.Priority, + UserId: opts.UserId, + CreateTs: time.Now(), + } + t.SetId(primitive.NewObjectID()) + t2, err := svc.schedulerSvc.Enqueue(t, opts.UserId) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.Id) + } + } else { + // single task + nodeIds, err := svc.getNodeIds(opts) + if err != nil { + return nil, err + } + if len(nodeIds) > 0 { + mainTask.NodeId = nodeIds[0] + } + t2, err := svc.schedulerSvc.Enqueue(mainTask, opts.UserId) + if err != nil { + return nil, err + } + taskIds = append(taskIds, t2.Id) + } + + return taskIds, nil +} + +func (svc *ServiceV2) getNodeIds(opts *interfaces.SpiderRunOptions) (nodeIds []primitive.ObjectID, err error) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + if err != nil { + return nil, err + } + for _, node := range nodes { + nodeIds = append(nodeIds, node.Id) + } + } else if opts.Mode == constants.RunTypeSelectedNodes { + nodeIds = opts.NodeIds + } + return nodeIds, nil +} + +func (svc *ServiceV2) isMultiTask(opts *interfaces.SpiderRunOptions) (res bool) { + if opts.Mode == constants.RunTypeAllNodes { + query := bson.M{ + "active": true, + "enabled": true, + "status": constants.NodeStatusOnline, + } + nodes, err := service.NewModelServiceV2[models.NodeV2]().GetMany(query, nil) + if err != nil { + trace.PrintError(err) + return false + } + return len(nodes) > 1 + } else if opts.Mode == constants.RunTypeRandom { + return false + } else if opts.Mode == constants.RunTypeSelectedNodes { + return len(opts.NodeIds) > 1 + } else { + return false + } +} + +func (svc *ServiceV2) syncGit() { + if svc.syncLock { + log.Infof("[SpiderAdminService] sync git is locked, skip") + return + } + log.Infof("[SpiderAdminService] start to sync git") + + svc.syncLock = true + defer func() { + svc.syncLock = false + }() + + // spiders + spiders, err := service.NewModelServiceV2[models.SpiderV2]().GetMany(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + + // spider ids + var spiderIds []primitive.ObjectID + for _, s := range spiders { + spiderIds = append(spiderIds, s.Id) + } + + if len(spiderIds) > 0 { + // gits + gits, err := service.NewModelServiceV2[models.GitV2]().GetMany(bson.M{ + "_id": bson.M{ + "$in": spiderIds, + }, + "auto_pull": true, + }, nil) + if err != nil { + trace.PrintError(err) + return + } + + wg := sync.WaitGroup{} + wg.Add(len(gits)) + for _, g := range gits { + go func(g models.GitV2) { + svc.syncGitOne(&g) + wg.Done() + }(g) + } + wg.Wait() + } + + log.Infof("[SpiderAdminService] finished sync git") +} + +func (svc *ServiceV2) syncGitOne(g *models.GitV2) { + log.Infof("[SpiderAdminService] sync git %s", g.Id) + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + // git client + workspacePath := viper.GetString("workspace") + gitClient, err := vcs.NewGitClient(vcs.WithPath(filepath.Join(workspacePath, g.Id.Hex()))) + if err != nil { + return + } + + // set auth + utils.InitGitClientAuthV2(g, gitClient) + + // check if remote has changes + ok, err := gitClient.IsRemoteChanged() + if err != nil { + trace.PrintError(err) + return + } + if !ok { + // no change + return + } + + // pull and sync to workspace + if err := gitClient.Reset(); err != nil { + trace.PrintError(err) + return + } + if err := gitClient.Pull(); err != nil { + trace.PrintError(err) + return + } + + // wait for context to end + <-ctx.Done() +} + +func NewSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { + svc := &ServiceV2{ + nodeCfgSvc: config.GetNodeConfigService(), + cfgPath: config2.GetConfigPath(), + } + svc.schedulerSvc, err = scheduler.GetTaskSchedulerServiceV2() + if err != nil { + return nil, err + } + + // cron + svc.cron = cron.New() + + // validate node type + if !svc.nodeCfgSvc.IsMaster() { + return nil, trace.TraceError(errors.ErrorSpiderForbidden) + } + + return svc, nil +} + +var svcV2 *ServiceV2 + +func GetSpiderAdminServiceV2() (svc2 *ServiceV2, err error) { + if svcV2 != nil { + return svcV2, nil + } + + svcV2, err = NewSpiderAdminServiceV2() + if err != nil { + return nil, err + } + + return svcV2, nil +} diff --git a/core/stats/options.go b/core/stats/options.go new file mode 100644 index 00000000..b40e16c3 --- /dev/null +++ b/core/stats/options.go @@ -0,0 +1,5 @@ +package stats + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(svc interfaces.StatsService) diff --git a/core/stats/service.go b/core/stats/service.go new file mode 100644 index 00000000..17df1be9 --- /dev/null +++ b/core/stats/service.go @@ -0,0 +1,323 @@ +package stats + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" +) + +type Service struct { +} + +func (svc *Service) GetOverviewStats(query bson.M) (data interface{}, err error) { + stats := bson.M{} + + // nodes + stats["nodes"], err = mongo.GetMongoCol(interfaces.ModelColNameNode).Count(bson.M{"active": true}) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["nodes"] = 0 + } + + // projects + stats["projects"], err = mongo.GetMongoCol(interfaces.ModelColNameProject).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["projects"] = 0 + } + + // spiders + stats["spiders"], err = mongo.GetMongoCol(interfaces.ModelColNameSpider).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["spiders"] = 0 + } + + // schedules + stats["schedules"], err = mongo.GetMongoCol(interfaces.ModelColNameSchedule).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["schedules"] = 0 + } + + // tasks + stats["tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["tasks"] = 0 + } + + // error tasks + stats["error_tasks"], err = mongo.GetMongoCol(interfaces.ModelColNameTask).Count(bson.M{"status": constants.TaskStatusError}) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["error_tasks"] = 0 + } + + // results + stats["results"], err = svc.getOverviewResults(query) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["results"] = 0 + } + + // users + stats["users"], err = mongo.GetMongoCol(interfaces.ModelColNameUser).Count(nil) + if err != nil { + if err.Error() != mongo2.ErrNoDocuments.Error() { + return nil, err + } + stats["users"] = 0 + } + + return stats, nil +} + +func (svc *Service) GetDailyStats(query bson.M) (data interface{}, err error) { + tasksStats, err := svc.getDailyTasksStats(query) + if err != nil { + return nil, err + } + return tasksStats, nil +} + +func (svc *Service) GetTaskStats(query bson.M) (data interface{}, err error) { + stats := bson.M{} + + // by status + stats["by_status"], err = svc.getTaskStatsByStatus(query) + if err != nil { + return nil, err + } + + // by node + stats["by_node"], err = svc.getTaskStatsByNode(query) + if err != nil { + return nil, err + } + + // by spider + stats["by_spider"], err = svc.getTaskStatsBySpider(query) + if err != nil { + return nil, err + } + + return stats, nil +} + +func (svc *Service) getDailyTasksStats(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{ + "$match", query, + }}, + {{ + "$addFields", + bson.M{ + "date": bson.M{ + "$dateToString": bson.M{ + "date": bson.M{"$toDate": "$_id"}, + "format": "%Y-%m-%d", + "timezone": "Asia/Shanghai", // TODO: parameterization + }, + }, + }, + }}, + {{ + "$group", + bson.M{ + "_id": "$date", + "tasks": bson.M{"$sum": 1}, + "results": bson.M{"$sum": "$result_count"}, + }, + }}, + {{ + "$sort", + bson.D{{"_id", 1}}, + }}, + } + var results []entity.StatsDailyItem + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getOverviewResults(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": nil, + "results": bson.M{"$sum": "$result_count"}, + }, + }}, + } + var res bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Aggregate(pipeline, nil).One(&res); err != nil { + return nil, err + } + return res["results"], nil +} + +func (svc *Service) getTaskStatsByStatus(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": "$status", + "tasks": bson.M{"$sum": 1}, + }, + }}, + {{ + "$project", + bson.M{ + "status": "$_id", + "tasks": "$tasks", + }, + }}, + } + var results []bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getTaskStatsByNode(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": "$node_id", + "tasks": bson.M{"$sum": 1}, + }, + }}, + {{ + "$lookup", + bson.M{ + "from": interfaces.ModelColNameNode, + "localField": "_id", + "foreignField": "_id", + "as": "_n", + }, + }}, + {{ + "$project", + bson.M{ + "node_id": "$node_id", + "node": bson.M{"$arrayElemAt": bson.A{"$_n", 0}}, + "node_name": bson.M{"$arrayElemAt": bson.A{"$_n.name", 0}}, + "tasks": "$tasks", + }, + }}, + } + var results []bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getTaskStatsBySpider(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$group", + bson.M{ + "_id": "$spider_id", + "tasks": bson.M{"$sum": 1}, + }, + }}, + {{ + "$lookup", + bson.M{ + "from": interfaces.ModelColNameSpider, + "localField": "_id", + "foreignField": "_id", + "as": "_s", + }, + }}, + {{ + "$project", + bson.M{ + "spider_id": "$spider_id", + "spider": bson.M{"$arrayElemAt": bson.A{"$_s", 0}}, + "spider_name": bson.M{"$arrayElemAt": bson.A{"$_s.name", 0}}, + "tasks": "$tasks", + }, + }}, + {{"$limit", 10}}, + } + var results []bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).All(&results); err != nil { + return nil, err + } + return results, nil +} + +func (svc *Service) getTaskStatsHistogram(query bson.M) (data interface{}, err error) { + pipeline := mongo2.Pipeline{ + {{"$match", query}}, + {{ + "$lookup", + bson.M{ + "from": interfaces.ModelColNameTaskStat, + "localField": "_id", + "foreignField": "_id", + "as": "_ts", + }, + }}, + {{ + "$facet", + bson.M{ + "total_duration": bson.A{ + bson.M{ + "$bucketAuto": bson.M{ + "groupBy": "$_ts.td", + "buckets": 10, + "granularity": "1-2-5", + }, + }, + }, + }, + }}, + } + var res bson.M + if err := mongo.GetMongoCol(interfaces.ModelColNameTask).Aggregate(pipeline, nil).One(&res); err != nil { + return nil, err + } + return res, nil +} + +var svc interfaces.StatsService + +func GetStatsService() interfaces.StatsService { + if svc != nil { + return svc + } + + // service + svc = &Service{} + + return svc +} diff --git a/core/sys_exec/sys_exec.go b/core/sys_exec/sys_exec.go new file mode 100644 index 00000000..397e4357 --- /dev/null +++ b/core/sys_exec/sys_exec.go @@ -0,0 +1,89 @@ +package sys_exec + +import ( + "bufio" + "github.com/crawlab-team/go-trace" + "github.com/shirou/gopsutil/process" + "os/exec" + "time" +) + +type KillProcessOptions struct { + Timeout time.Duration + Force bool +} + +func KillProcess(cmd *exec.Cmd, opts *KillProcessOptions) error { + // process + p, err := process.NewProcess(int32(cmd.Process.Pid)) + if err != nil { + return err + } + + // kill function + killFunc := func(p *process.Process) error { + return killProcessRecursive(p, opts.Force) + } + + if opts.Timeout != 0 { + // with timeout + return killProcessWithTimeout(p, opts.Timeout, killFunc) + } else { + // without timeout + return killFunc(p) + } +} + +func killProcessWithTimeout(p *process.Process, timeout time.Duration, killFunc func(*process.Process) error) error { + go func() { + if err := killFunc(p); err != nil { + trace.PrintError(err) + } + }() + for i := 0; i < int(timeout.Seconds()); i++ { + ok, err := process.PidExists(p.Pid) + if err == nil && !ok { + return nil + } + time.Sleep(1 * time.Second) + } + return killProcess(p, true) +} + +func killProcessRecursive(p *process.Process, force bool) (err error) { + // children processes + cps, err := p.Children() + if err != nil { + return killProcess(p, force) + } + + // iterate children processes + for _, cp := range cps { + if err := killProcessRecursive(cp, force); err != nil { + return err + } + } + + return nil +} + +func killProcess(p *process.Process, force bool) (err error) { + if force { + err = p.Kill() + } else { + err = p.Terminate() + } + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func ConfigureCmdLogging(cmd *exec.Cmd, fn func(scanner *bufio.Scanner)) { + stdout, _ := (*cmd).StdoutPipe() + stderr, _ := (*cmd).StderrPipe() + scannerStdout := bufio.NewScanner(stdout) + scannerStderr := bufio.NewScanner(stderr) + go fn(scannerStdout) + go fn(scannerStderr) +} diff --git a/core/sys_exec/sys_exec_darwin.go b/core/sys_exec/sys_exec_darwin.go new file mode 100644 index 00000000..130b7cb8 --- /dev/null +++ b/core/sys_exec/sys_exec_darwin.go @@ -0,0 +1,24 @@ +//go:build darwin +// +build darwin + +package sys_exec + +import ( + "os/exec" + "syscall" +) + +func BuildCmd(cmdStr string) *exec.Cmd { + return exec.Command("sh", "-c", cmdStr) +} + +func SetPgid(cmd *exec.Cmd) { + if cmd == nil { + return + } + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } else { + cmd.SysProcAttr.Setpgid = true + } +} diff --git a/core/sys_exec/sys_exec_linux.go b/core/sys_exec/sys_exec_linux.go new file mode 100644 index 00000000..bf532a43 --- /dev/null +++ b/core/sys_exec/sys_exec_linux.go @@ -0,0 +1,24 @@ +//go:build linux +// +build linux + +package sys_exec + +import ( + "os/exec" + "syscall" +) + +func BuildCmd(cmdStr string) *exec.Cmd { + return exec.Command("sh", "-c", cmdStr) +} + +func SetPgid(cmd *exec.Cmd) { + if cmd == nil { + return + } + if cmd.SysProcAttr == nil { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } else { + cmd.SysProcAttr.Setpgid = true + } +} diff --git a/core/sys_exec/sys_exec_windows.go b/core/sys_exec/sys_exec_windows.go new file mode 100644 index 00000000..e2678ccd --- /dev/null +++ b/core/sys_exec/sys_exec_windows.go @@ -0,0 +1,10 @@ +//go:build windows +// +build windows + +package sys_exec + +import "os/exec" + +func BuildCmd(cmdStr string) *exec.Cmd { + return exec.Command("cmd", "/C", cmdStr) +} diff --git a/core/system/service.go b/core/system/service.go new file mode 100644 index 00000000..10846740 --- /dev/null +++ b/core/system/service.go @@ -0,0 +1,86 @@ +package system + +import ( + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +type Service struct { + col *mongo2.Col + modelSvc service.ModelService +} + +func (svc *Service) Init() (err error) { + // initialize data + if err := svc.initData(); err != nil { + return err + } + + return nil +} + +func (svc *Service) initData() (err error) { + total, err := svc.col.Count(bson.M{ + "key": "site_title", + }) + if err != nil { + return err + } + if total > 0 { + return nil + } + + // data to initialize + settings := []models.Setting{ + { + Id: primitive.NewObjectID(), + Key: "site_title", + Value: bson.M{ + "customize_site_title": false, + "site_title": "", + }, + }, + } + var data []interface{} + for _, s := range settings { + data = append(data, s) + } + _, err = svc.col.InsertMany(data) + if err != nil { + return err + } + return nil +} + +func NewService() *Service { + // service + svc := &Service{ + col: mongo2.GetMongoCol(interfaces.ModelColNameSetting), + } + + // model service + modelSvc, err := service.GetService() + if err != nil { + panic(err) + } + svc.modelSvc = modelSvc + + if err := svc.Init(); err != nil { + panic(err) + } + + return svc +} + +var _service *Service + +func GetService() *Service { + if _service == nil { + _service = NewService() + } + return _service +} diff --git a/core/task/base.go b/core/task/base.go new file mode 100644 index 00000000..265186a3 --- /dev/null +++ b/core/task/base.go @@ -0,0 +1,97 @@ +package task + +import ( + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" +) + +type BaseService struct { + // dependencies + interfaces.WithConfigPath + modelSvc service.ModelService + + // internals + stopped bool +} + +func (svc *BaseService) Init() error { + // implement me + return nil +} + +func (svc *BaseService) Start() { + // implement me +} + +func (svc *BaseService) Wait() { + utils.DefaultWait() +} + +func (svc *BaseService) Stop() { + svc.stopped = true +} + +// SaveTask deprecated +func (svc *BaseService) SaveTask(t interfaces.Task, status string) (err error) { + // normalize status + if status == "" { + status = constants.TaskStatusPending + } + + // set task status + t.SetStatus(status) + + // attempt to get task from database + _, err = svc.modelSvc.GetTaskById(t.GetId()) + if err != nil { + // if task does not exist, add to database + if err == mongo.ErrNoDocuments { + if err := delegate.NewModelDelegate(t).Add(); err != nil { + return err + } + return nil + } else { + return err + } + } else { + // otherwise, update + if err := delegate.NewModelDelegate(t).Save(); err != nil { + return err + } + return nil + } +} + +func (svc *BaseService) IsStopped() (res bool) { + return svc.stopped +} + +func (svc *BaseService) GetQueue(nodeId primitive.ObjectID) (queue string) { + if nodeId.IsZero() { + return fmt.Sprintf("%s", constants.TaskListQueuePrefixPublic) + } else { + return fmt.Sprintf("%s:%s", constants.TaskListQueuePrefixNodes, nodeId.Hex()) + } +} + +func NewBaseService() (svc2 interfaces.TaskBaseService, err error) { + svc := &BaseService{} + + // dependency injection + if err := container.GetContainer().Invoke(func(cfgPath interfaces.WithConfigPath, modelSvc service.ModelService) { + svc.WithConfigPath = cfgPath + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} diff --git a/core/task/handler/options.go b/core/task/handler/options.go new file mode 100644 index 00000000..c2c03558 --- /dev/null +++ b/core/task/handler/options.go @@ -0,0 +1,40 @@ +package handler + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.TaskHandlerService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetConfigPath(path) + } +} + +func WithExitWatchDuration(duration time.Duration) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetExitWatchDuration(duration) + } +} + +func WithReportInterval(interval time.Duration) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetReportInterval(interval) + } +} + +func WithCancelTimeout(timeout time.Duration) Option { + return func(svc interfaces.TaskHandlerService) { + svc.SetCancelTimeout(timeout) + } +} + +type RunnerOption func(r interfaces.TaskRunner) + +func WithSubscribeTimeout(timeout time.Duration) RunnerOption { + return func(r interfaces.TaskRunner) { + r.SetSubscribeTimeout(timeout) + } +} diff --git a/core/task/handler/runner.go b/core/task/handler/runner.go new file mode 100644 index 00000000..11930910 --- /dev/null +++ b/core/task/handler/runner.go @@ -0,0 +1,691 @@ +package handler + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + fs2 "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/shirou/gopsutil/process" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type Runner struct { + // dependencies + svc interfaces.TaskHandlerService // task handler service + fsSvc interfaces.FsServiceV2 // task fs service + hookSvc interfaces.TaskHookService // task hook service + + // settings + subscribeTimeout time.Duration + bufferSize int + + // internals + cmd *exec.Cmd // process command instance + pid int // process id + tid primitive.ObjectID // task id + t interfaces.Task // task model.Task + s interfaces.Spider // spider model.Spider + ch chan constants.TaskSignal // channel to communicate between Service and Runner + err error // standard process error + envs []models.Env // environment variables + cwd string // working directory + c interfaces.GrpcClient // grpc client + sub grpc.TaskService_SubscribeClient // grpc task service stream client + + // log internals + scannerStdout *bufio.Reader + scannerStderr *bufio.Reader + logBatchSize int +} + +func (r *Runner) Init() (err error) { + // update task + if err := r.updateTask("", nil); err != nil { + return err + } + + // start grpc client + if !r.c.IsStarted() { + r.c.Start() + } + + // working directory + workspacePath := viper.GetString("workspace") + r.cwd = filepath.Join(workspacePath, r.s.GetId().Hex()) + + // sync files from master + if !utils.IsMaster() { + if err := r.syncFiles(); err != nil { + return err + } + } + + // grpc task service stream client + if err := r.initSub(); err != nil { + return err + } + + // pre actions + if r.hookSvc != nil { + if err := r.hookSvc.PreActions(r.t, r.s, r.fsSvc, r.svc); err != nil { + return err + } + } + + return nil +} + +func (r *Runner) Run() (err error) { + // log task started + log.Infof("task[%s] started", r.tid.Hex()) + + // configure cmd + r.configureCmd() + + // configure environment variables + r.configureEnv() + + // configure logging + r.configureLogging() + + // start process + if err := r.cmd.Start(); err != nil { + return r.updateTask(constants.TaskStatusError, err) + } + + // start logging + go r.startLogging() + + // process id + if r.cmd.Process == nil { + return r.updateTask(constants.TaskStatusError, constants.ErrNotExists) + } + r.pid = r.cmd.Process.Pid + r.t.SetPid(r.pid) + + // update task status (processing) + if err := r.updateTask(constants.TaskStatusRunning, nil); err != nil { + return err + } + + // wait for process to finish + go r.wait() + + // start health check + go r.startHealthCheck() + + // declare task status + status := "" + + // wait for signal + signal := <-r.ch + switch signal { + case constants.TaskSignalFinish: + err = nil + status = constants.TaskStatusFinished + case constants.TaskSignalCancel: + err = constants.ErrTaskCancelled + status = constants.TaskStatusCancelled + case constants.TaskSignalError: + err = r.err + status = constants.TaskStatusError + case constants.TaskSignalLost: + err = constants.ErrTaskLost + status = constants.TaskStatusError + default: + err = constants.ErrInvalidSignal + status = constants.TaskStatusError + } + + // update task status + if err := r.updateTask(status, err); err != nil { + return err + } + + // post actions + if r.hookSvc != nil { + if err := r.hookSvc.PostActions(r.t, r.s, r.fsSvc, r.svc); err != nil { + return err + } + } + + return err +} + +func (r *Runner) Cancel() (err error) { + // kill process + opts := &sys_exec.KillProcessOptions{ + Timeout: r.svc.GetCancelTimeout(), + Force: true, + } + if err := sys_exec.KillProcess(r.cmd, opts); err != nil { + return err + } + + // make sure the process does not exist + op := func() error { + if exists, _ := process.PidExists(int32(r.pid)); exists { + return errors.ErrorTaskProcessStillExists + } + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), r.svc.GetExitWatchDuration()) + defer cancel() + b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) + if err := backoff.Retry(op, b); err != nil { + return trace.TraceError(errors.ErrorTaskUnableToCancel) + } + + return nil +} + +// CleanUp clean up task runner +func (r *Runner) CleanUp() (err error) { + return nil +} + +func (r *Runner) SetSubscribeTimeout(timeout time.Duration) { + r.subscribeTimeout = timeout +} + +func (r *Runner) GetTaskId() (id primitive.ObjectID) { + return r.tid +} + +func (r *Runner) configureCmd() { + var cmdStr string + + // customized spider + if r.t.GetCmd() == "" { + cmdStr = r.s.GetCmd() + } else { + cmdStr = r.t.GetCmd() + } + + // parameters + if r.t.GetParam() != "" { + cmdStr += " " + r.t.GetParam() + } else if r.s.GetParam() != "" { + cmdStr += " " + r.s.GetParam() + } + + // get cmd instance + r.cmd = sys_exec.BuildCmd(cmdStr) + + // set working directory + r.cmd.Dir = r.cwd + + // configure pgid to allow killing sub processes + //sys_exec.SetPgid(r.cmd) +} + +func (r *Runner) configureLogging() { + // set stdout reader + stdout, _ := r.cmd.StdoutPipe() + r.scannerStdout = bufio.NewReaderSize(stdout, r.bufferSize) + + // set stderr reader + stderr, _ := r.cmd.StderrPipe() + r.scannerStderr = bufio.NewReaderSize(stderr, r.bufferSize) +} + +func (r *Runner) startLogging() { + // start reading stdout + go r.startLoggingReaderStdout() + + // start reading stderr + go r.startLoggingReaderStderr() +} + +func (r *Runner) startLoggingReaderStdout() { + for { + line, err := r.scannerStdout.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *Runner) startLoggingReaderStderr() { + for { + line, err := r.scannerStderr.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *Runner) startHealthCheck() { + if r.cmd.ProcessState == nil || r.cmd.ProcessState.Exited() { + return + } + for { + exists, _ := process.PidExists(int32(r.pid)) + if !exists { + // process lost + r.ch <- constants.TaskSignalLost + return + } + time.Sleep(1 * time.Second) + } +} + +func (r *Runner) configureEnv() { + // 默认把Node.js的全局node_modules加入环境变量 + envPath := os.Getenv("PATH") + nodePath := "/usr/lib/node_modules" + if !strings.Contains(envPath, nodePath) { + _ = os.Setenv("PATH", nodePath+":"+envPath) + } + _ = os.Setenv("NODE_PATH", nodePath) + + // default envs + r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex()) + if viper.GetString("grpc.address") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address")) + } + if viper.GetString("grpc.authKey") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey")) + } else { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+constants.DefaultGrpcAuthKey) + } + + // global environment variables + envs, err := r.svc.GetModelEnvironmentService().GetEnvironmentList(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + for _, env := range envs { + r.cmd.Env = append(r.cmd.Env, env.GetKey()+"="+env.GetValue()) + } +} + +func (r *Runner) syncFiles() (err error) { + masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.GetId().Hex()) + workspacePath := viper.GetString("workspace") + workerDir := filepath.Join(workspacePath, r.s.GetId().Hex()) + + // get file list from master + resp, err := http.Get(masterURL + "/scan") + if err != nil { + fmt.Println("Error getting file list from master:", err) + return trace.TraceError(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return trace.TraceError(err) + } + var masterFiles map[string]entity.FsFileInfo + err = json.Unmarshal(body, &masterFiles) + if err != nil { + fmt.Println("Error unmarshaling JSON:", err) + return trace.TraceError(err) + } + + // create a map for master files + masterFilesMap := make(map[string]entity.FsFileInfo) + for _, file := range masterFiles { + masterFilesMap[file.Path] = file + } + + // create worker directory if not exists + if _, err := os.Stat(workerDir); os.IsNotExist(err) { + if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { + fmt.Println("Error creating worker directory:", err) + return trace.TraceError(err) + } + } + + // get file list from worker + workerFiles, err := utils.ScanDirectory(workerDir) + if err != nil { + fmt.Println("Error scanning worker directory:", err) + return trace.TraceError(err) + } + + // set up wait group and error channel + var wg sync.WaitGroup + errCh := make(chan error, 1) + + // delete files that are deleted on master node + for path, workerFile := range workerFiles { + if _, exists := masterFilesMap[path]; !exists { + fmt.Println("Deleting file:", path) + err := os.Remove(workerFile.FullPath) + if err != nil { + fmt.Println("Error deleting file:", err) + } + } + } + + // download files that are new or modified on master node + for path, masterFile := range masterFilesMap { + workerFile, exists := workerFiles[path] + if !exists || masterFile.Hash != workerFile.Hash { + wg.Add(1) + go func(path string, masterFile entity.FsFileInfo) { + defer wg.Done() + logrus.Infof("File needs to be synchronized: %s", path) + err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) + if err != nil { + logrus.Errorf("Error downloading file: %v", err) + select { + case errCh <- err: + default: + } + } + }(path, masterFile) + } + } + + wg.Wait() + close(errCh) + if err := <-errCh; err != nil { + return err + } + + return nil +} + +func (r *Runner) downloadFile(url string, filePath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + out, err := os.Create(filePath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// wait for process to finish and send task signal (constants.TaskSignal) +// to task runner's channel (Runner.ch) according to exit code +func (r *Runner) wait() { + // wait for process to finish + if err := r.cmd.Wait(); err != nil { + exitError, ok := err.(*exec.ExitError) + if !ok { + r.ch <- constants.TaskSignalError + return + } + exitCode := exitError.ExitCode() + if exitCode == -1 { + // cancel error + r.ch <- constants.TaskSignalCancel + return + } + + // standard error + r.err = err + r.ch <- constants.TaskSignalError + return + } + + // success + r.ch <- constants.TaskSignalFinish +} + +// updateTask update and get updated info of task (Runner.t) +func (r *Runner) updateTask(status string, e error) (err error) { + if r.t != nil && status != "" { + // update task status + r.t.SetStatus(status) + if e != nil { + r.t.SetError(e.Error()) + } + if r.svc.GetNodeConfigService().IsMaster() { + if err := delegate.NewModelDelegate(r.t).Save(); err != nil { + return err + } + } else { + if err := client.NewModelDelegate(r.t, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { + return err + } + } + + // send notification + go r.sendNotification() + + // update stats + go func() { + r._updateTaskStat(status) + r._updateSpiderStat(status) + }() + } + + // get task + r.t, err = r.svc.GetTaskById(r.tid) + if err != nil { + return err + } + + return nil +} + +func (r *Runner) initSub() (err error) { + r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func (r *Runner) writeLogLines(lines []string) { + data, err := json.Marshal(&entity.StreamMessageTaskData{ + TaskId: r.tid, + Logs: lines, + }) + if err != nil { + trace.PrintError(err) + return + } + msg := &grpc.StreamMessage{ + Code: grpc.StreamMessageCode_INSERT_LOGS, + Data: data, + } + if err := r.sub.Send(msg); err != nil { + trace.PrintError(err) + return + } +} + +func (r *Runner) _updateTaskStat(status string) { + ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + switch status { + case constants.TaskStatusPending: + // do nothing + case constants.TaskStatusRunning: + ts.SetStartTs(time.Now()) + ts.SetWaitDuration(ts.GetStartTs().Sub(ts.GetCreateTs()).Milliseconds()) + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + ts.SetEndTs(time.Now()) + ts.SetRuntimeDuration(ts.GetEndTs().Sub(ts.GetStartTs()).Milliseconds()) + ts.SetTotalDuration(ts.GetEndTs().Sub(ts.GetCreateTs()).Milliseconds()) + } + if r.svc.GetNodeConfigService().IsMaster() { + if err := delegate.NewModelDelegate(ts).Save(); err != nil { + trace.PrintError(err) + return + } + } else { + if err := client.NewModelDelegate(ts, client.WithDelegateConfigPath(r.svc.GetConfigPath())).Save(); err != nil { + trace.PrintError(err) + return + } + } +} + +func (r *Runner) sendNotification() { + data, err := json.Marshal(r.t) + if err != nil { + trace.PrintError(err) + return + } + req := &grpc.Request{ + NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), + Data: data, + } + _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) + if err != nil { + trace.PrintError(err) + return + } +} + +func (r *Runner) _updateSpiderStat(status string) { + // task stat + ts, err := r.svc.GetModelTaskStatService().GetTaskStatById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + + // update + var update bson.M + switch status { + case constants.TaskStatusPending, constants.TaskStatusRunning: + update = bson.M{ + "$set": bson.M{ + "last_task_id": r.tid, // last task id + }, + "$inc": bson.M{ + "tasks": 1, // task count + "wait_duration": ts.GetWaitDuration(), // wait duration + }, + } + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + update = bson.M{ + "$inc": bson.M{ + "results": ts.GetResultCount(), // results + "runtime_duration": ts.GetRuntimeDuration() / 1000, // runtime duration + "total_duration": ts.GetTotalDuration() / 1000, // total duration + }, + } + default: + trace.PrintError(errors.ErrorTaskInvalidType) + return + } + + // perform update + if r.svc.GetNodeConfigService().IsMaster() { + if err := mongo.GetMongoCol(interfaces.ModelColNameSpiderStat).UpdateId(r.s.GetId(), update); err != nil { + trace.PrintError(err) + return + } + } else { + modelSvc, err := client.NewBaseServiceDelegate( + client.WithBaseServiceModelId(interfaces.ModelIdSpiderStat), + client.WithBaseServiceConfigPath(r.svc.GetConfigPath()), + ) + if err != nil { + trace.PrintError(err) + return + } + if err := modelSvc.UpdateById(r.s.GetId(), update); err != nil { + trace.PrintError(err) + return + } + } + +} + +func NewTaskRunner(id primitive.ObjectID, svc interfaces.TaskHandlerService, opts ...RunnerOption) (r2 interfaces.TaskRunner, err error) { + // validate options + if id.IsZero() { + return nil, constants.ErrInvalidOptions + } + + // runner + r := &Runner{ + subscribeTimeout: 30 * time.Second, + bufferSize: 1024 * 1024, + svc: svc, + tid: id, + ch: make(chan constants.TaskSignal), + logBatchSize: 20, + } + + // apply options + for _, opt := range opts { + opt(r) + } + + // task + r.t, err = svc.GetTaskById(id) + if err != nil { + return nil, err + } + + // spider + r.s, err = svc.GetSpiderById(r.t.GetSpiderId()) + if err != nil { + return nil, err + } + + // task fs service + r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.GetId().Hex())) + + // dependency injection + if err := container.GetContainer().Invoke(func( + c interfaces.GrpcClient, + ) { + r.c = c + }); err != nil { + return nil, trace.TraceError(err) + } + + _ = container.GetContainer().Invoke(func(hookSvc interfaces.TaskHookService) { + r.hookSvc = hookSvc + }) + + // initialize task runner + if err := r.Init(); err != nil { + return r, err + } + + return r, nil +} diff --git a/core/task/handler/runner_test.go b/core/task/handler/runner_test.go new file mode 100644 index 00000000..175117a3 --- /dev/null +++ b/core/task/handler/runner_test.go @@ -0,0 +1,103 @@ +package handler + +import ( + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/google/uuid" + "github.com/spf13/viper" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockRunner struct { + mock.Mock + Runner +} + +func (m *MockRunner) downloadFile(url string, filePath string) error { + args := m.Called(url, filePath) + return args.Error(0) +} + +func newMockRunner() *MockRunner { + r := &MockRunner{} + r.s = &models.Spider{} + return r +} + +func TestSyncFiles_SuccessWithDummyFiles(t *testing.T) { + // Create a test server that responds with a list of files + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/scan") { + w.Write([]byte(`{"file1.txt":{"path": "file1.txt", "hash": "hash1"}, "file2.txt":{"path": "file2.txt", "hash": "hash2"}}`)) + return + } + + if strings.HasSuffix(r.URL.Path, "/download") { + w.Write([]byte("file content")) + return + } + })) + defer ts.Close() + + // Create a mock runner + r := newMockRunner() + r.On("downloadFile", mock.Anything, mock.Anything).Return(nil) + + // Set the master URL to the test server URL + viper.Set("api.endpoint", ts.URL) + + localPath := filepath.Join(os.TempDir(), uuid.New().String()) + os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm) + defer os.RemoveAll(localPath) + viper.Set("workspace", localPath) + + // Call the method under test + err := r.syncFiles() + assert.NoError(t, err) + + // Assert that the files were downloaded + assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file1.txt")) + assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file2.txt")) +} + +func TestSyncFiles_DeletesFilesNotOnMaster(t *testing.T) { + // Create a test server that responds with an empty list of files + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/scan") { + w.Write([]byte(`{}`)) + return + } + })) + defer ts.Close() + + // Create a mock runner + r := newMockRunner() + r.On("downloadFile", mock.Anything, mock.Anything).Return(nil) + + // Set the master URL to the test server URL + viper.Set("api.endpoint", ts.URL) + + localPath := filepath.Join(os.TempDir(), uuid.New().String()) + os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm) + defer os.RemoveAll(localPath) + viper.Set("workspace", localPath) + + // Create a dummy file that should be deleted + dummyFilePath := filepath.Join(localPath, r.s.GetId().Hex(), "dummy.txt") + dummyFile, _ := os.Create(dummyFilePath) + dummyFile.Close() + + // Call the method under test + err := r.syncFiles() + assert.NoError(t, err) + + // Assert that the dummy file was deleted + assert.NoFileExists(t, dummyFilePath) +} diff --git a/core/task/handler/runner_v2.go b/core/task/handler/runner_v2.go new file mode 100644 index 00000000..5818f25c --- /dev/null +++ b/core/task/handler/runner_v2.go @@ -0,0 +1,671 @@ +package handler + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/crawlab/core/errors" + fs2 "github.com/crawlab-team/crawlab/core/fs" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/models" + service2 "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "github.com/shirou/gopsutil/process" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" +) + +type RunnerV2 struct { + // dependencies + svc *ServiceV2 // task handler service + fsSvc interfaces.FsServiceV2 // task fs service + + // settings + subscribeTimeout time.Duration + bufferSize int + + // internals + cmd *exec.Cmd // process command instance + pid int // process id + tid primitive.ObjectID // task id + t *models.TaskV2 // task model.Task + s *models.SpiderV2 // spider model.Spider + ch chan constants.TaskSignal // channel to communicate between Service and RunnerV2 + err error // standard process error + envs []models.Env // environment variables + cwd string // working directory + c interfaces.GrpcClient // grpc client + sub grpc.TaskService_SubscribeClient // grpc task service stream client + + // log internals + scannerStdout *bufio.Reader + scannerStderr *bufio.Reader + logBatchSize int +} + +func (r *RunnerV2) Init() (err error) { + // update task + if err := r.updateTask("", nil); err != nil { + return err + } + + // start grpc client + if !r.c.IsStarted() { + r.c.Start() + } + + // working directory + workspacePath := viper.GetString("workspace") + r.cwd = filepath.Join(workspacePath, r.s.Id.Hex()) + + // sync files from master + if !utils.IsMaster() { + if err := r.syncFiles(); err != nil { + return err + } + } + + // grpc task service stream client + if err := r.initSub(); err != nil { + return err + } + + return nil +} + +func (r *RunnerV2) Run() (err error) { + // log task started + log.Infof("task[%s] started", r.tid.Hex()) + + // configure cmd + r.configureCmd() + + // configure environment variables + r.configureEnv() + + // configure logging + r.configureLogging() + + // start process + if err := r.cmd.Start(); err != nil { + return r.updateTask(constants.TaskStatusError, err) + } + + // start logging + go r.startLogging() + + // process id + if r.cmd.Process == nil { + return r.updateTask(constants.TaskStatusError, constants.ErrNotExists) + } + r.pid = r.cmd.Process.Pid + r.t.Pid = r.pid + + // update task status (processing) + if err := r.updateTask(constants.TaskStatusRunning, nil); err != nil { + return err + } + + // wait for process to finish + go r.wait() + + // start health check + go r.startHealthCheck() + + // declare task status + status := "" + + // wait for signal + signal := <-r.ch + switch signal { + case constants.TaskSignalFinish: + err = nil + status = constants.TaskStatusFinished + case constants.TaskSignalCancel: + err = constants.ErrTaskCancelled + status = constants.TaskStatusCancelled + case constants.TaskSignalError: + err = r.err + status = constants.TaskStatusError + case constants.TaskSignalLost: + err = constants.ErrTaskLost + status = constants.TaskStatusError + default: + err = constants.ErrInvalidSignal + status = constants.TaskStatusError + } + + // update task status + if err := r.updateTask(status, err); err != nil { + return err + } + + return err +} + +func (r *RunnerV2) Cancel() (err error) { + // kill process + opts := &sys_exec.KillProcessOptions{ + Timeout: r.svc.GetCancelTimeout(), + Force: true, + } + if err := sys_exec.KillProcess(r.cmd, opts); err != nil { + return err + } + + // make sure the process does not exist + op := func() error { + if exists, _ := process.PidExists(int32(r.pid)); exists { + return errors.ErrorTaskProcessStillExists + } + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), r.svc.GetExitWatchDuration()) + defer cancel() + b := backoff.WithContext(backoff.NewConstantBackOff(1*time.Second), ctx) + if err := backoff.Retry(op, b); err != nil { + return trace.TraceError(errors.ErrorTaskUnableToCancel) + } + + return nil +} + +// CleanUp clean up task runner +func (r *RunnerV2) CleanUp() (err error) { + return nil +} + +func (r *RunnerV2) SetSubscribeTimeout(timeout time.Duration) { + r.subscribeTimeout = timeout +} + +func (r *RunnerV2) GetTaskId() (id primitive.ObjectID) { + return r.tid +} + +func (r *RunnerV2) configureCmd() { + var cmdStr string + + // customized spider + if r.t.Cmd == "" { + cmdStr = r.s.Cmd + } else { + cmdStr = r.t.Cmd + } + + // parameters + if r.t.Param != "" { + cmdStr += " " + r.t.Param + } else if r.s.Param != "" { + cmdStr += " " + r.s.Param + } + + // get cmd instance + r.cmd = sys_exec.BuildCmd(cmdStr) + + // set working directory + r.cmd.Dir = r.cwd + + // configure pgid to allow killing sub processes + //sys_exec.SetPgid(r.cmd) +} + +func (r *RunnerV2) configureLogging() { + // set stdout reader + stdout, _ := r.cmd.StdoutPipe() + r.scannerStdout = bufio.NewReaderSize(stdout, r.bufferSize) + + // set stderr reader + stderr, _ := r.cmd.StderrPipe() + r.scannerStderr = bufio.NewReaderSize(stderr, r.bufferSize) +} + +func (r *RunnerV2) startLogging() { + // start reading stdout + go r.startLoggingReaderStdout() + + // start reading stderr + go r.startLoggingReaderStderr() +} + +func (r *RunnerV2) startLoggingReaderStdout() { + for { + line, err := r.scannerStdout.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *RunnerV2) startLoggingReaderStderr() { + for { + line, err := r.scannerStderr.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + r.writeLogLines([]string{line}) + } +} + +func (r *RunnerV2) startHealthCheck() { + if r.cmd.ProcessState == nil || r.cmd.ProcessState.Exited() { + return + } + for { + exists, _ := process.PidExists(int32(r.pid)) + if !exists { + // process lost + r.ch <- constants.TaskSignalLost + return + } + time.Sleep(1 * time.Second) + } +} + +func (r *RunnerV2) configureEnv() { + // 默认把Node.js的全局node_modules加入环境变量 + envPath := os.Getenv("PATH") + nodePath := "/usr/lib/node_modules" + if !strings.Contains(envPath, nodePath) { + _ = os.Setenv("PATH", nodePath+":"+envPath) + } + _ = os.Setenv("NODE_PATH", nodePath) + + // default envs + r.cmd.Env = append(os.Environ(), "CRAWLAB_TASK_ID="+r.tid.Hex()) + if viper.GetString("grpc.address") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_ADDRESS="+viper.GetString("grpc.address")) + } + if viper.GetString("grpc.authKey") != "" { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+viper.GetString("grpc.authKey")) + } else { + r.cmd.Env = append(r.cmd.Env, "CRAWLAB_GRPC_AUTH_KEY="+constants.DefaultGrpcAuthKey) + } + + // global environment variables + envs, err := client.NewModelServiceV2[models.EnvironmentV2]().GetMany(nil, nil) + if err != nil { + trace.PrintError(err) + return + } + for _, env := range envs { + r.cmd.Env = append(r.cmd.Env, env.Key+"="+env.Value) + } +} + +func (r *RunnerV2) syncFiles() (err error) { + masterURL := fmt.Sprintf("%s/sync/%s", viper.GetString("api.endpoint"), r.s.Id.Hex()) + workspacePath := viper.GetString("workspace") + workerDir := filepath.Join(workspacePath, r.s.Id.Hex()) + + // get file list from master + resp, err := http.Get(masterURL + "/scan") + if err != nil { + fmt.Println("Error getting file list from master:", err) + return trace.TraceError(err) + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return trace.TraceError(err) + } + var masterFiles map[string]entity.FsFileInfo + err = json.Unmarshal(body, &masterFiles) + if err != nil { + fmt.Println("Error unmarshaling JSON:", err) + return trace.TraceError(err) + } + + // create a map for master files + masterFilesMap := make(map[string]entity.FsFileInfo) + for _, file := range masterFiles { + masterFilesMap[file.Path] = file + } + + // create worker directory if not exists + if _, err := os.Stat(workerDir); os.IsNotExist(err) { + if err := os.MkdirAll(workerDir, os.ModePerm); err != nil { + fmt.Println("Error creating worker directory:", err) + return trace.TraceError(err) + } + } + + // get file list from worker + workerFiles, err := utils.ScanDirectory(workerDir) + if err != nil { + fmt.Println("Error scanning worker directory:", err) + return trace.TraceError(err) + } + + // set up wait group and error channel + var wg sync.WaitGroup + errCh := make(chan error, 1) + + // delete files that are deleted on master node + for path, workerFile := range workerFiles { + if _, exists := masterFilesMap[path]; !exists { + fmt.Println("Deleting file:", path) + err := os.Remove(workerFile.FullPath) + if err != nil { + fmt.Println("Error deleting file:", err) + } + } + } + + // download files that are new or modified on master node + for path, masterFile := range masterFilesMap { + workerFile, exists := workerFiles[path] + if !exists || masterFile.Hash != workerFile.Hash { + wg.Add(1) + go func(path string, masterFile entity.FsFileInfo) { + defer wg.Done() + logrus.Infof("File needs to be synchronized: %s", path) + err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(workerDir, path)) + if err != nil { + logrus.Errorf("Error downloading file: %v", err) + select { + case errCh <- err: + default: + } + } + }(path, masterFile) + } + } + + wg.Wait() + close(errCh) + if err := <-errCh; err != nil { + return err + } + + return nil +} + +func (r *RunnerV2) downloadFile(url string, filePath string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + out, err := os.Create(filePath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} + +// wait for process to finish and send task signal (constants.TaskSignal) +// to task runner's channel (RunnerV2.ch) according to exit code +func (r *RunnerV2) wait() { + // wait for process to finish + if err := r.cmd.Wait(); err != nil { + exitError, ok := err.(*exec.ExitError) + if !ok { + r.ch <- constants.TaskSignalError + return + } + exitCode := exitError.ExitCode() + if exitCode == -1 { + // cancel error + r.ch <- constants.TaskSignalCancel + return + } + + // standard error + r.err = err + r.ch <- constants.TaskSignalError + return + } + + // success + r.ch <- constants.TaskSignalFinish +} + +// updateTask update and get updated info of task (RunnerV2.t) +func (r *RunnerV2) updateTask(status string, e error) (err error) { + if r.t != nil && status != "" { + // update task status + r.t.Status = status + if e != nil { + r.t.Error = e.Error() + } + if r.svc.GetNodeConfigService().IsMaster() { + err = service2.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + if err != nil { + return err + } + } else { + err = client.NewModelServiceV2[models.TaskV2]().ReplaceById(r.t.Id, *r.t) + if err != nil { + return err + } + } + + // send notification + go r.sendNotification() + + // update stats + go func() { + r._updateTaskStat(status) + r._updateSpiderStat(status) + }() + } + + // get task + r.t, err = r.svc.GetTaskById(r.tid) + if err != nil { + return err + } + + return nil +} + +func (r *RunnerV2) initSub() (err error) { + r.sub, err = r.c.GetTaskClient().Subscribe(context.Background()) + if err != nil { + return trace.TraceError(err) + } + return nil +} + +func (r *RunnerV2) writeLogLines(lines []string) { + data, err := json.Marshal(&entity.StreamMessageTaskData{ + TaskId: r.tid, + Logs: lines, + }) + if err != nil { + trace.PrintError(err) + return + } + msg := &grpc.StreamMessage{ + Code: grpc.StreamMessageCode_INSERT_LOGS, + Data: data, + } + if err := r.sub.Send(msg); err != nil { + trace.PrintError(err) + return + } +} + +func (r *RunnerV2) _updateTaskStat(status string) { + ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + switch status { + case constants.TaskStatusPending: + // do nothing + case constants.TaskStatusRunning: + ts.StartTs = time.Now() + ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + if ts.StartTs.IsZero() { + ts.StartTs = time.Now() + ts.WaitDuration = ts.StartTs.Sub(ts.CreateTs).Milliseconds() + } + ts.EndTs = time.Now() + ts.RuntimeDuration = ts.EndTs.Sub(ts.StartTs).Milliseconds() + ts.TotalDuration = ts.EndTs.Sub(ts.CreateTs).Milliseconds() + } + if r.svc.GetNodeConfigService().IsMaster() { + err = service2.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + if err != nil { + trace.PrintError(err) + return + } + } else { + err = client.NewModelServiceV2[models.TaskStatV2]().ReplaceById(ts.Id, *ts) + if err != nil { + trace.PrintError(err) + return + } + } +} + +func (r *RunnerV2) sendNotification() { + data, err := json.Marshal(r.t) + if err != nil { + trace.PrintError(err) + return + } + req := &grpc.Request{ + NodeKey: r.svc.GetNodeConfigService().GetNodeKey(), + Data: data, + } + _, err = r.c.GetTaskClient().SendNotification(context.Background(), req) + if err != nil { + trace.PrintError(err) + return + } +} + +func (r *RunnerV2) _updateSpiderStat(status string) { + // task stat + ts, err := client.NewModelServiceV2[models.TaskStatV2]().GetById(r.tid) + if err != nil { + trace.PrintError(err) + return + } + + // update + var update bson.M + switch status { + case constants.TaskStatusPending, constants.TaskStatusRunning: + update = bson.M{ + "$set": bson.M{ + "last_task_id": r.tid, // last task id + }, + "$inc": bson.M{ + "tasks": 1, // task count + "wait_duration": ts.WaitDuration, // wait duration + }, + } + case constants.TaskStatusFinished, constants.TaskStatusError, constants.TaskStatusCancelled: + update = bson.M{ + "$set": bson.M{ + "last_task_id": r.tid, // last task id + }, + "$inc": bson.M{ + "results": ts.ResultCount, // results + "runtime_duration": ts.RuntimeDuration / 1000, // runtime duration + "total_duration": ts.TotalDuration / 1000, // total duration + }, + } + default: + trace.PrintError(errors.ErrorTaskInvalidType) + return + } + + // perform update + if r.svc.GetNodeConfigService().IsMaster() { + err = service2.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + if err != nil { + trace.PrintError(err) + return + } + } else { + err = client.NewModelServiceV2[models.SpiderStatV2]().UpdateById(r.s.Id, update) + if err != nil { + trace.PrintError(err) + return + } + } + +} + +func NewTaskRunnerV2(id primitive.ObjectID, svc *ServiceV2) (r2 *RunnerV2, err error) { + // validate options + if id.IsZero() { + return nil, constants.ErrInvalidOptions + } + + // runner + r := &RunnerV2{ + subscribeTimeout: 30 * time.Second, + bufferSize: 1024 * 1024, + svc: svc, + tid: id, + ch: make(chan constants.TaskSignal), + logBatchSize: 20, + } + + // task + r.t, err = svc.GetTaskById(id) + if err != nil { + return nil, err + } + + // spider + r.s, err = svc.GetSpiderById(r.t.SpiderId) + if err != nil { + return nil, err + } + + // task fs service + r.fsSvc = fs2.NewFsServiceV2(filepath.Join(viper.GetString("workspace"), r.s.Id.Hex())) + + // dependency injection + if err := container.GetContainer().Invoke(func( + c interfaces.GrpcClient, + ) { + r.c = c + }); err != nil { + return nil, trace.TraceError(err) + } + + // initialize task runner + if err := r.Init(); err != nil { + return r, err + } + + return r, nil +} diff --git a/core/task/handler/service.go b/core/task/handler/service.go new file mode 100644 index 00000000..84a929ad --- /dev/null +++ b/core/task/handler/service.go @@ -0,0 +1,506 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + errors2 "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/task" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "sync" + "time" +) + +type Service struct { + // dependencies + interfaces.TaskBaseService + cfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + clientModelSvc interfaces.GrpcClientModelService + clientModelNodeSvc interfaces.GrpcClientModelNodeService + clientModelSpiderSvc interfaces.GrpcClientModelSpiderService + clientModelTaskSvc interfaces.GrpcClientModelTaskService + clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService + clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService + c interfaces.GrpcClient // grpc client + + // settings + //maxRunners int + exitWatchDuration time.Duration + reportInterval time.Duration + fetchInterval time.Duration + fetchTimeout time.Duration + cancelTimeout time.Duration + + // internals variables + stopped bool + mu sync.Mutex + runners sync.Map // pool of task runners started + syncLocks sync.Map // files sync locks map of task runners +} + +func (svc *Service) Start() { + // Initialize gRPC if not started + if !svc.c.IsStarted() { + svc.c.Start() + } + + go svc.ReportStatus() + go svc.Fetch() +} + +func (svc *Service) Run(taskId primitive.ObjectID) (err error) { + return svc.run(taskId) +} + +func (svc *Service) Reset() { + svc.mu.Lock() + defer svc.mu.Unlock() +} + +func (svc *Service) Cancel(taskId primitive.ObjectID) (err error) { + r, err := svc.getRunner(taskId) + if err != nil { + return err + } + if err := r.Cancel(); err != nil { + return err + } + return nil +} + +func (svc *Service) Fetch() { + for { + // wait + time.Sleep(svc.fetchInterval) + + // current node + n, err := svc.GetCurrentNode() + if err != nil { + continue + } + + // skip if node is not active or enabled + if !n.GetActive() || !n.GetEnabled() { + continue + } + + // validate if there are available runners + if svc.getRunnerCount() >= n.GetMaxRunners() { + continue + } + + // stop + if svc.stopped { + return + } + + // fetch task + tid, err := svc.fetch() + if err != nil { + trace.PrintError(err) + continue + } + + // skip if no task id + if tid.IsZero() { + continue + } + + // run task + if err := svc.run(tid); err != nil { + trace.PrintError(err) + t, err := svc.GetTaskById(tid) + if err == nil && t.GetStatus() != constants.TaskStatusCancelled { + t.SetError(err.Error()) + _ = svc.SaveTask(t, constants.TaskStatusError) + continue + } + continue + } + } +} + +func (svc *Service) ReportStatus() { + for { + if svc.stopped { + return + } + + // report handler status + if err := svc.reportStatus(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.reportInterval) + } +} + +func (svc *Service) IsSyncLocked(path string) (ok bool) { + _, ok = svc.syncLocks.Load(path) + return ok +} + +func (svc *Service) LockSync(path string) { + svc.syncLocks.Store(path, true) +} + +func (svc *Service) UnlockSync(path string) { + svc.syncLocks.Delete(path) +} + +//func (svc *Service) GetMaxRunners() (maxRunners int) { +// return svc.maxRunners +//} +// +//func (svc *Service) SetMaxRunners(maxRunners int) { +// svc.maxRunners = maxRunners +//} + +func (svc *Service) GetExitWatchDuration() (duration time.Duration) { + return svc.exitWatchDuration +} + +func (svc *Service) SetExitWatchDuration(duration time.Duration) { + svc.exitWatchDuration = duration +} + +func (svc *Service) GetFetchInterval() (interval time.Duration) { + return svc.fetchInterval +} + +func (svc *Service) SetFetchInterval(interval time.Duration) { + svc.fetchInterval = interval +} + +func (svc *Service) GetReportInterval() (interval time.Duration) { + return svc.reportInterval +} + +func (svc *Service) SetReportInterval(interval time.Duration) { + svc.reportInterval = interval +} + +func (svc *Service) GetCancelTimeout() (timeout time.Duration) { + return svc.cancelTimeout +} + +func (svc *Service) SetCancelTimeout(timeout time.Duration) { + svc.cancelTimeout = timeout +} + +func (svc *Service) GetModelService() (modelSvc interfaces.GrpcClientModelService) { + return svc.clientModelSvc +} + +func (svc *Service) GetModelSpiderService() (modelSpiderSvc interfaces.GrpcClientModelSpiderService) { + return svc.clientModelSpiderSvc +} + +func (svc *Service) GetModelTaskService() (modelTaskSvc interfaces.GrpcClientModelTaskService) { + return svc.clientModelTaskSvc +} + +func (svc *Service) GetModelTaskStatService() (modelTaskSvc interfaces.GrpcClientModelTaskStatService) { + return svc.clientModelTaskStatSvc +} + +func (svc *Service) GetModelEnvironmentService() (modelTaskSvc interfaces.GrpcClientModelEnvironmentService) { + return svc.clientModelEnvironmentSvc +} + +func (svc *Service) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *Service) GetCurrentNode() (n interfaces.Node, err error) { + // node key + nodeKey := svc.cfgSvc.GetNodeKey() + + // current node + if svc.cfgSvc.IsMaster() { + n, err = svc.modelSvc.GetNodeByKey(nodeKey, nil) + } else { + n, err = svc.clientModelNodeSvc.GetNodeByKey(nodeKey) + } + if err != nil { + return nil, err + } + + return n, nil +} + +func (svc *Service) GetTaskById(id primitive.ObjectID) (t interfaces.Task, err error) { + if svc.cfgSvc.IsMaster() { + t, err = svc.modelSvc.GetTaskById(id) + } else { + t, err = svc.clientModelTaskSvc.GetTaskById(id) + } + if err != nil { + return nil, err + } + + return t, nil +} + +func (svc *Service) GetSpiderById(id primitive.ObjectID) (s interfaces.Spider, err error) { + if svc.cfgSvc.IsMaster() { + s, err = svc.modelSvc.GetSpiderById(id) + } else { + s, err = svc.clientModelSpiderSvc.GetSpiderById(id) + } + if err != nil { + return nil, err + } + + return s, nil +} + +func (svc *Service) getRunners() (runners []*Runner) { + svc.mu.Lock() + defer svc.mu.Unlock() + svc.runners.Range(func(key, value interface{}) bool { + r := value.(Runner) + runners = append(runners, &r) + return true + }) + return runners +} + +func (svc *Service) getRunnerCount() (count int) { + n, err := svc.GetCurrentNode() + if err != nil { + trace.PrintError(err) + return + } + query := bson.M{ + "node_id": n.GetId(), + "status": constants.TaskStatusRunning, + } + if svc.cfgSvc.IsMaster() { + count, err = svc.modelSvc.GetBaseService(interfaces.ModelIdTask).Count(query) + if err != nil { + trace.PrintError(err) + return + } + } else { + count, err = svc.clientModelTaskSvc.Count(query) + if err != nil { + trace.PrintError(err) + return + } + } + return count +} + +func (svc *Service) getRunner(taskId primitive.ObjectID) (r interfaces.TaskRunner, err error) { + log.Debugf("[TaskHandlerService] getRunner: taskId[%v]", taskId) + v, ok := svc.runners.Load(taskId) + if !ok { + return nil, trace.TraceError(errors2.ErrorTaskNotExists) + } + switch v.(type) { + case interfaces.TaskRunner: + r = v.(interfaces.TaskRunner) + default: + return nil, trace.TraceError(errors2.ErrorModelInvalidType) + } + return r, nil +} + +func (svc *Service) addRunner(taskId primitive.ObjectID, r interfaces.TaskRunner) { + log.Debugf("[TaskHandlerService] addRunner: taskId[%v]", taskId) + svc.runners.Store(taskId, r) +} + +func (svc *Service) deleteRunner(taskId primitive.ObjectID) { + log.Debugf("[TaskHandlerService] deleteRunner: taskId[%v]", taskId) + svc.runners.Delete(taskId) +} + +func (svc *Service) saveTask(t interfaces.Task, status string) (err error) { + // normalize status + if status == "" { + status = constants.TaskStatusPending + } + + // set task status + t.SetStatus(status) + + // attempt to get task from database + _, err = svc.clientModelTaskSvc.GetTaskById(t.GetId()) + if err != nil { + // if task does not exist, add to database + if err == mongo.ErrNoDocuments { + if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Add(); err != nil { + return err + } + return nil + } else { + return err + } + } else { + // otherwise, update + if err := client.NewModelDelegate(t, client.WithDelegateConfigPath(svc.GetConfigPath())).Save(); err != nil { + return err + } + return nil + } +} + +func (svc *Service) reportStatus() (err error) { + // current node + n, err := svc.GetCurrentNode() + if err != nil { + return err + } + + // available runners of handler + ar := n.GetMaxRunners() - svc.getRunnerCount() + + // set available runners + n.SetAvailableRunners(ar) + + // save node + if svc.cfgSvc.IsMaster() { + err = delegate.NewModelDelegate(n).Save() + } else { + err = client.NewModelDelegate(n, client.WithDelegateConfigPath(svc.GetConfigPath())).Save() + } + if err != nil { + return err + } + + return nil +} + +func (svc *Service) fetch() (tid primitive.ObjectID, err error) { + ctx, cancel := context.WithTimeout(context.Background(), svc.fetchTimeout) + defer cancel() + res, err := svc.c.GetTaskClient().Fetch(ctx, svc.c.NewRequest(nil)) + if err != nil { + return tid, trace.TraceError(err) + } + if err := json.Unmarshal(res.Data, &tid); err != nil { + return tid, trace.TraceError(err) + } + return tid, nil +} + +func (svc *Service) run(taskId primitive.ObjectID) (err error) { + // attempt to get runner from pool + _, ok := svc.runners.Load(taskId) + if ok { + return trace.TraceError(errors2.ErrorTaskAlreadyExists) + } + + // create a new task runner + r, err := NewTaskRunner(taskId, svc) + if err != nil { + return trace.TraceError(err) + } + + // add runner to pool + svc.addRunner(taskId, r) + + // create a goroutine to run task + go func() { + // delete runner from pool + defer svc.deleteRunner(r.GetTaskId()) + defer func(r interfaces.TaskRunner) { + err := r.CleanUp() + if err != nil { + log.Errorf("task[%s] clean up error: %v", r.GetTaskId().Hex(), err) + } + }(r) + // run task process (blocking) + // error or finish after task runner ends + if err := r.Run(); err != nil { + switch { + case errors.Is(err, constants.ErrTaskError): + log.Errorf("task[%s] finished with error: %v", r.GetTaskId().Hex(), err) + case errors.Is(err, constants.ErrTaskCancelled): + log.Errorf("task[%s] cancelled", r.GetTaskId().Hex()) + default: + log.Errorf("task[%s] finished with unknown error: %v", r.GetTaskId().Hex(), err) + } + } + log.Infof("task[%s] finished", r.GetTaskId().Hex()) + }() + + return nil +} + +func NewTaskHandlerService() (svc2 interfaces.TaskHandlerService, err error) { + // base service + baseSvc, err := task.NewBaseService() + if err != nil { + return nil, trace.TraceError(err) + } + + // service + svc := &Service{ + TaskBaseService: baseSvc, + exitWatchDuration: 60 * time.Second, + fetchInterval: 1 * time.Second, + fetchTimeout: 15 * time.Second, + reportInterval: 5 * time.Second, + cancelTimeout: 5 * time.Second, + mu: sync.Mutex{}, + runners: sync.Map{}, + syncLocks: sync.Map{}, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + cfgSvc interfaces.NodeConfigService, + modelSvc service.ModelService, + clientModelSvc interfaces.GrpcClientModelService, + clientModelNodeSvc interfaces.GrpcClientModelNodeService, + clientModelSpiderSvc interfaces.GrpcClientModelSpiderService, + clientModelTaskSvc interfaces.GrpcClientModelTaskService, + clientModelTaskStatSvc interfaces.GrpcClientModelTaskStatService, + clientModelEnvironmentSvc interfaces.GrpcClientModelEnvironmentService, + c interfaces.GrpcClient, + ) { + svc.cfgSvc = cfgSvc + svc.modelSvc = modelSvc + svc.clientModelSvc = clientModelSvc + svc.clientModelNodeSvc = clientModelNodeSvc + svc.clientModelSpiderSvc = clientModelSpiderSvc + svc.clientModelTaskSvc = clientModelTaskSvc + svc.clientModelTaskStatSvc = clientModelTaskStatSvc + svc.clientModelEnvironmentSvc = clientModelEnvironmentSvc + svc.c = c + }); err != nil { + return nil, trace.TraceError(err) + } + + log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) + + return svc, nil +} + +var _service interfaces.TaskHandlerService + +func GetTaskHandlerService() (svr interfaces.TaskHandlerService, err error) { + if _service != nil { + return _service, nil + } + _service, err = NewTaskHandlerService() + if err != nil { + return nil, err + } + return _service, nil +} diff --git a/core/task/handler/service_v2.go b/core/task/handler/service_v2.go new file mode 100644 index 00000000..397d5f11 --- /dev/null +++ b/core/task/handler/service_v2.go @@ -0,0 +1,426 @@ +package handler + +import ( + "context" + "encoding/json" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + errors2 "github.com/crawlab-team/crawlab/core/errors" + grpcclient "github.com/crawlab-team/crawlab/core/grpc/client" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/client" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + cfgSvc interfaces.NodeConfigService + c *grpcclient.GrpcClientV2 // grpc client + + // settings + //maxRunners int + exitWatchDuration time.Duration + reportInterval time.Duration + fetchInterval time.Duration + fetchTimeout time.Duration + cancelTimeout time.Duration + + // internals variables + stopped bool + mu sync.Mutex + runners sync.Map // pool of task runners started + syncLocks sync.Map // files sync locks map of task runners +} + +func (svc *ServiceV2) Start() { + // Initialize gRPC if not started + if !svc.c.IsStarted() { + svc.c.Start() + } + + go svc.ReportStatus() + go svc.Fetch() +} + +func (svc *ServiceV2) Run(taskId primitive.ObjectID) (err error) { + return svc.run(taskId) +} + +func (svc *ServiceV2) Reset() { + svc.mu.Lock() + defer svc.mu.Unlock() +} + +func (svc *ServiceV2) Cancel(taskId primitive.ObjectID) (err error) { + r, err := svc.getRunner(taskId) + if err != nil { + return err + } + if err := r.Cancel(); err != nil { + return err + } + return nil +} + +func (svc *ServiceV2) Fetch() { + for { + // wait + time.Sleep(svc.fetchInterval) + + // current node + n, err := svc.GetCurrentNode() + if err != nil { + continue + } + + // skip if node is not active or enabled + if !n.Active || !n.Enabled { + continue + } + + // validate if there are available runners + if svc.getRunnerCount() >= n.MaxRunners { + continue + } + + // stop + if svc.stopped { + return + } + + // fetch task + tid, err := svc.fetch() + if err != nil { + trace.PrintError(err) + continue + } + + // skip if no task id + if tid.IsZero() { + continue + } + + // run task + if err := svc.run(tid); err != nil { + trace.PrintError(err) + t, err := svc.GetTaskById(tid) + if err == nil && t.Status != constants.TaskStatusCancelled { + t.Error = err.Error() + t.Status = constants.TaskStatusError + t.SetUpdated(t.CreatedBy) + continue + } + continue + } + } +} + +func (svc *ServiceV2) ReportStatus() { + for { + if svc.stopped { + return + } + + // report handler status + if err := svc.reportStatus(); err != nil { + trace.PrintError(err) + } + + // wait + time.Sleep(svc.reportInterval) + } +} + +func (svc *ServiceV2) IsSyncLocked(path string) (ok bool) { + _, ok = svc.syncLocks.Load(path) + return ok +} + +func (svc *ServiceV2) LockSync(path string) { + svc.syncLocks.Store(path, true) +} + +func (svc *ServiceV2) UnlockSync(path string) { + svc.syncLocks.Delete(path) +} + +//func (svc *ServiceV2) GetMaxRunners() (maxRunners int) { +// return svc.maxRunners +//} +// +//func (svc *ServiceV2) SetMaxRunners(maxRunners int) { +// svc.maxRunners = maxRunners +//} + +func (svc *ServiceV2) GetExitWatchDuration() (duration time.Duration) { + return svc.exitWatchDuration +} + +func (svc *ServiceV2) SetExitWatchDuration(duration time.Duration) { + svc.exitWatchDuration = duration +} + +func (svc *ServiceV2) GetFetchInterval() (interval time.Duration) { + return svc.fetchInterval +} + +func (svc *ServiceV2) SetFetchInterval(interval time.Duration) { + svc.fetchInterval = interval +} + +func (svc *ServiceV2) GetReportInterval() (interval time.Duration) { + return svc.reportInterval +} + +func (svc *ServiceV2) SetReportInterval(interval time.Duration) { + svc.reportInterval = interval +} + +func (svc *ServiceV2) GetCancelTimeout() (timeout time.Duration) { + return svc.cancelTimeout +} + +func (svc *ServiceV2) SetCancelTimeout(timeout time.Duration) { + svc.cancelTimeout = timeout +} + +func (svc *ServiceV2) GetNodeConfigService() (cfgSvc interfaces.NodeConfigService) { + return svc.cfgSvc +} + +func (svc *ServiceV2) GetCurrentNode() (n *models.NodeV2, err error) { + // node key + nodeKey := svc.cfgSvc.GetNodeKey() + + // current node + if svc.cfgSvc.IsMaster() { + n, err = service.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + } else { + n, err = client.NewModelServiceV2[models.NodeV2]().GetOne(bson.M{"key": nodeKey}, nil) + } + if err != nil { + return nil, err + } + + return n, nil +} + +func (svc *ServiceV2) GetTaskById(id primitive.ObjectID) (t *models.TaskV2, err error) { + if svc.cfgSvc.IsMaster() { + t, err = service.NewModelServiceV2[models.TaskV2]().GetById(id) + } else { + t, err = client.NewModelServiceV2[models.TaskV2]().GetById(id) + } + if err != nil { + return nil, err + } + + return t, nil +} + +func (svc *ServiceV2) GetSpiderById(id primitive.ObjectID) (s *models.SpiderV2, err error) { + if svc.cfgSvc.IsMaster() { + s, err = service.NewModelServiceV2[models.SpiderV2]().GetById(id) + } else { + s, err = client.NewModelServiceV2[models.SpiderV2]().GetById(id) + } + if err != nil { + return nil, err + } + + return s, nil +} + +func (svc *ServiceV2) getRunners() (runners []*Runner) { + svc.mu.Lock() + defer svc.mu.Unlock() + svc.runners.Range(func(key, value interface{}) bool { + r := value.(Runner) + runners = append(runners, &r) + return true + }) + return runners +} + +func (svc *ServiceV2) getRunnerCount() (count int) { + n, err := svc.GetCurrentNode() + if err != nil { + trace.PrintError(err) + return + } + query := bson.M{ + "node_id": n.Id, + "status": constants.TaskStatusRunning, + } + if svc.cfgSvc.IsMaster() { + count, err = service.NewModelServiceV2[models.TaskV2]().Count(query) + if err != nil { + trace.PrintError(err) + return + } + } else { + count, err = client.NewModelServiceV2[models.TaskV2]().Count(query) + if err != nil { + trace.PrintError(err) + return + } + } + return count +} + +func (svc *ServiceV2) getRunner(taskId primitive.ObjectID) (r interfaces.TaskRunner, err error) { + log.Debugf("[TaskHandlerService] getRunner: taskId[%v]", taskId) + v, ok := svc.runners.Load(taskId) + if !ok { + return nil, trace.TraceError(errors2.ErrorTaskNotExists) + } + switch v.(type) { + case interfaces.TaskRunner: + r = v.(interfaces.TaskRunner) + default: + return nil, trace.TraceError(errors2.ErrorModelInvalidType) + } + return r, nil +} + +func (svc *ServiceV2) addRunner(taskId primitive.ObjectID, r interfaces.TaskRunner) { + log.Debugf("[TaskHandlerService] addRunner: taskId[%v]", taskId) + svc.runners.Store(taskId, r) +} + +func (svc *ServiceV2) deleteRunner(taskId primitive.ObjectID) { + log.Debugf("[TaskHandlerService] deleteRunner: taskId[%v]", taskId) + svc.runners.Delete(taskId) +} + +func (svc *ServiceV2) reportStatus() (err error) { + // current node + n, err := svc.GetCurrentNode() + if err != nil { + return err + } + + // available runners of handler + ar := n.MaxRunners - svc.getRunnerCount() + + // set available runners + n.AvailableRunners = ar + + // save node + n.SetUpdated(n.CreatedBy) + if svc.cfgSvc.IsMaster() { + err = service.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + } else { + err = client.NewModelServiceV2[models.NodeV2]().ReplaceById(n.Id, *n) + } + if err != nil { + return err + } + + return nil +} + +func (svc *ServiceV2) fetch() (tid primitive.ObjectID, err error) { + ctx, cancel := context.WithTimeout(context.Background(), svc.fetchTimeout) + defer cancel() + res, err := svc.c.TaskClient.Fetch(ctx, svc.c.NewRequest(nil)) + if err != nil { + return tid, trace.TraceError(err) + } + if err := json.Unmarshal(res.Data, &tid); err != nil { + return tid, trace.TraceError(err) + } + return tid, nil +} + +func (svc *ServiceV2) run(taskId primitive.ObjectID) (err error) { + // attempt to get runner from pool + _, ok := svc.runners.Load(taskId) + if ok { + return trace.TraceError(errors2.ErrorTaskAlreadyExists) + } + + // create a new task runner + r, err := NewTaskRunnerV2(taskId, svc) + if err != nil { + return trace.TraceError(err) + } + + // add runner to pool + svc.addRunner(taskId, r) + + // create a goroutine to run task + go func() { + // delete runner from pool + defer svc.deleteRunner(r.GetTaskId()) + defer func(r interfaces.TaskRunner) { + err := r.CleanUp() + if err != nil { + log.Errorf("task[%s] clean up error: %v", r.GetTaskId().Hex(), err) + } + }(r) + // run task process (blocking) + // error or finish after task runner ends + if err := r.Run(); err != nil { + switch { + case errors.Is(err, constants.ErrTaskError): + log.Errorf("task[%s] finished with error: %v", r.GetTaskId().Hex(), err) + case errors.Is(err, constants.ErrTaskCancelled): + log.Errorf("task[%s] cancelled", r.GetTaskId().Hex()) + default: + log.Errorf("task[%s] finished with unknown error: %v", r.GetTaskId().Hex(), err) + } + } + log.Infof("task[%s] finished", r.GetTaskId().Hex()) + }() + + return nil +} + +func NewTaskHandlerServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + exitWatchDuration: 60 * time.Second, + fetchInterval: 1 * time.Second, + fetchTimeout: 15 * time.Second, + reportInterval: 5 * time.Second, + cancelTimeout: 5 * time.Second, + mu: sync.Mutex{}, + runners: sync.Map{}, + syncLocks: sync.Map{}, + } + + // dependency injection + svc.cfgSvc = nodeconfig.GetNodeConfigService() + + // grpc client + svc.c, err = grpcclient.NewGrpcClientV2() + if err != nil { + return nil, err + } + + log.Debugf("[NewTaskHandlerService] svc[cfgPath: %s]", svc.cfgSvc.GetConfigPath()) + + return svc, nil +} + +var _serviceV2 *ServiceV2 + +func GetTaskHandlerServiceV2() (svr *ServiceV2, err error) { + if _serviceV2 != nil { + return _serviceV2, nil + } + _serviceV2, err = NewTaskHandlerServiceV2() + if err != nil { + return nil, err + } + return _serviceV2, nil +} diff --git a/core/task/log/constants.go b/core/task/log/constants.go new file mode 100644 index 00000000..73676851 --- /dev/null +++ b/core/task/log/constants.go @@ -0,0 +1,12 @@ +package log + +const ( + MetadataName = "metadata.json" +) + +const ( + DriverTypeFile = "file" // raw file + DriverTypeFs = "fs" // file system (SeaweedFS) + DriverTypeMongo = "mongo" // mongodb + DriverTypeEs = "es" // elastic search +) diff --git a/core/task/log/default.go b/core/task/log/default.go new file mode 100644 index 00000000..2ac37f81 --- /dev/null +++ b/core/task/log/default.go @@ -0,0 +1,5 @@ +package log + +import "time" + +var DefaultLogTtl = 30 * 24 * time.Hour diff --git a/core/task/log/driver.go b/core/task/log/driver.go new file mode 100644 index 00000000..139ec44d --- /dev/null +++ b/core/task/log/driver.go @@ -0,0 +1,18 @@ +package log + +func GetLogDriver(logDriverType string) (driver Driver, err error) { + switch logDriverType { + case DriverTypeFile: + driver, err = GetFileLogDriver() + if err != nil { + return driver, err + } + case DriverTypeMongo: + return driver, ErrNotImplemented + case DriverTypeEs: + return driver, ErrNotImplemented + default: + return driver, ErrInvalidType + } + return driver, nil +} diff --git a/core/task/log/entity.go b/core/task/log/entity.go new file mode 100644 index 00000000..0583e41c --- /dev/null +++ b/core/task/log/entity.go @@ -0,0 +1,16 @@ +package log + +import "time" + +type Message struct { + Id int64 `json:"id" bson:"id"` + Msg string `json:"msg" bson:"msg"` + Ts time.Time `json:"ts" bson:"ts"` +} + +type Metadata struct { + Size int64 `json:"size,omitempty" bson:"size"` + TotalLines int64 `json:"total_lines,omitempty" bson:"total_lines"` + TotalBytes int64 `json:"total_bytes,omitempty" bson:"total_bytes"` + Md5 string `json:"md5,omitempty" bson:"md5"` +} diff --git a/core/task/log/errors.go b/core/task/log/errors.go new file mode 100644 index 00000000..eb7c1540 --- /dev/null +++ b/core/task/log/errors.go @@ -0,0 +1,8 @@ +package log + +import "errors" + +var ( + ErrInvalidType = errors.New("invalid type") + ErrNotImplemented = errors.New("not implemented") +) diff --git a/core/task/log/file_driver.go b/core/task/log/file_driver.go new file mode 100644 index 00000000..590a1c9a --- /dev/null +++ b/core/task/log/file_driver.go @@ -0,0 +1,284 @@ +package log + +import ( + "bufio" + "bytes" + "errors" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type FileLogDriver struct { + // settings + logFileName string + rootPath string + + // internals + mu sync.Mutex +} + +func (d *FileLogDriver) Init() (err error) { + go d.cleanup() + + return nil +} + +func (d *FileLogDriver) Close() (err error) { + return nil +} + +func (d *FileLogDriver) WriteLine(id string, line string) (err error) { + d.initDir(id) + + d.mu.Lock() + defer d.mu.Unlock() + filePath := d.getLogFilePath(id, d.logFileName) + + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0760)) + if err != nil { + return trace.TraceError(err) + } + defer func(f *os.File) { + err := f.Close() + if err != nil { + log.Errorf("close file error: %s", err.Error()) + } + }(f) + + _, err = f.WriteString(line + "\n") + if err != nil { + return trace.TraceError(err) + } + + return nil +} + +func (d *FileLogDriver) WriteLines(id string, lines []string) (err error) { + linesString := strings.Join(lines, "\n") + if err := d.WriteLine(id, linesString); err != nil { + return err + } + return nil +} + +func (d *FileLogDriver) Find(id string, pattern string, skip int, limit int) (lines []string, err error) { + if pattern != "" { + return lines, errors.New("not implemented") + } + if !utils.Exists(d.getLogFilePath(id, d.logFileName)) { + return nil, nil + } + + f, err := os.Open(d.getLogFilePath(id, d.logFileName)) + if err != nil { + return nil, trace.TraceError(err) + } + defer f.Close() + + sc := bufio.NewReaderSize(f, 1024*1024*10) + + i := -1 + for { + line, err := sc.ReadString(byte('\n')) + if err != nil { + break + } + line = strings.TrimSuffix(line, "\n") + + i++ + + if i < skip { + continue + } + + if i >= skip+limit { + break + } + + lines = append(lines, line) + } + + return lines, nil +} + +func (d *FileLogDriver) Count(id string, pattern string) (n int, err error) { + if pattern != "" { + return n, errors.New("not implemented") + } + if !utils.Exists(d.getLogFilePath(id, d.logFileName)) { + return 0, nil + } + + f, err := os.Open(d.getLogFilePath(id, d.logFileName)) + if err != nil { + return n, trace.TraceError(err) + } + return d.lineCounter(f) +} + +func (d *FileLogDriver) Flush() (err error) { + return nil +} + +func (d *FileLogDriver) getLogPath() (logPath string) { + return viper.GetString("log.path") +} + +func (d *FileLogDriver) getBasePath(id string) (filePath string) { + return filepath.Join(d.getLogPath(), id) +} + +func (d *FileLogDriver) getMetadataPath(id string) (filePath string) { + return filepath.Join(d.getBasePath(id), MetadataName) +} + +func (d *FileLogDriver) getLogFilePath(id, fileName string) (filePath string) { + return filepath.Join(d.getBasePath(id), fileName) +} + +func (d *FileLogDriver) getLogFiles(id string) (files []os.FileInfo) { + // 增加了对返回异常的捕获 + files, err := utils.ListDir(d.getBasePath(id)) + if err != nil { + trace.PrintError(err) + return nil + } + return +} + +func (d *FileLogDriver) initDir(id string) { + if !utils.Exists(d.getBasePath(id)) { + if err := os.MkdirAll(d.getBasePath(id), os.FileMode(0770)); err != nil { + trace.PrintError(err) + } + } +} + +func (d *FileLogDriver) lineCounter(r io.Reader) (n int, err error) { + buf := make([]byte, 32*1024) + count := 0 + lineSep := []byte{'\n'} + + for { + c, err := r.Read(buf) + count += bytes.Count(buf[:c], lineSep) + + switch { + case err == io.EOF: + return count, nil + + case err != nil: + return count, err + } + } +} + +func (d *FileLogDriver) getTtl() time.Duration { + ttl := viper.GetString("log.ttl") + if ttl == "" { + return DefaultLogTtl + } + + if strings.HasSuffix(ttl, "s") { + ttl = strings.TrimSuffix(ttl, "s") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * time.Second + } else if strings.HasSuffix(ttl, "m") { + ttl = strings.TrimSuffix(ttl, "m") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * time.Minute + } else if strings.HasSuffix(ttl, "h") { + ttl = strings.TrimSuffix(ttl, "h") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * time.Hour + + } else if strings.HasSuffix(ttl, "d") { + ttl = strings.TrimSuffix(ttl, "d") + n, err := strconv.Atoi(ttl) + if err != nil { + return DefaultLogTtl + } + return time.Duration(n) * 24 * time.Hour + } else { + return DefaultLogTtl + } +} + +func (d *FileLogDriver) cleanup() { + if d.getLogPath() == "" { + return + } + if !utils.Exists(d.getLogPath()) { + if err := os.MkdirAll(d.getLogPath(), os.FileMode(0770)); err != nil { + log.Errorf("failed to create log directory: %s", d.getLogPath()) + trace.PrintError(err) + return + } + } + for { + // 增加对目录不存在的判断 + dirs, err := utils.ListDir(d.getLogPath()) + if err != nil { + trace.PrintError(err) + time.Sleep(10 * time.Minute) + continue + } + for _, dir := range dirs { + if time.Now().After(dir.ModTime().Add(d.getTtl())) { + if err := os.RemoveAll(d.getBasePath(dir.Name())); err != nil { + trace.PrintError(err) + continue + } + log.Infof("removed outdated log directory: %s", d.getBasePath(dir.Name())) + } + } + + time.Sleep(10 * time.Minute) + } +} + +var logDriver Driver + +func newFileLogDriver() (driver Driver, err error) { + // driver + driver = &FileLogDriver{ + logFileName: "log.txt", + mu: sync.Mutex{}, + } + + // init + if err := driver.Init(); err != nil { + return nil, err + } + + return driver, nil +} + +func GetFileLogDriver() (driver Driver, err error) { + if logDriver != nil { + return logDriver, nil + } + logDriver, err = newFileLogDriver() + if err != nil { + return nil, err + } + return logDriver, nil +} diff --git a/core/task/log/file_driver_test.go b/core/task/log/file_driver_test.go new file mode 100644 index 00000000..260edf71 --- /dev/null +++ b/core/task/log/file_driver_test.go @@ -0,0 +1,132 @@ +package log + +import ( + "fmt" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/bson/primitive" + "os" + "strings" + "testing" +) + +func setupFileDriverTest() { + cleanupFileDriverTest() + _ = os.MkdirAll("./tmp", os.ModePerm) +} + +func cleanupFileDriverTest() { + _ = os.RemoveAll("./tmp") +} + +func TestFileDriver_WriteLine(t *testing.T) { + setupFileDriverTest() + t.Cleanup(cleanupFileDriverTest) + + d, err := newFileLogDriver(nil) + require.Nil(t, err) + defer d.Close() + + id := primitive.NewObjectID() + + err = d.WriteLine(id.Hex(), "it works") + require.Nil(t, err) + + logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex()) + require.FileExists(t, logFilePath) + text, err := os.ReadFile(logFilePath) + require.Nil(t, err) + require.Equal(t, "it works\n", string(text)) +} + +func TestFileDriver_WriteLines(t *testing.T) { + setupFileDriverTest() + t.Cleanup(cleanupFileDriverTest) + + d, err := newFileLogDriver(nil) + require.Nil(t, err) + defer d.Close() + + id := primitive.NewObjectID() + + for i := 0; i < 100; i++ { + err = d.WriteLine(id.Hex(), "it works") + require.Nil(t, err) + } + + logFilePath := fmt.Sprintf("/var/log/crawlab/%s/log.txt", id.Hex()) + require.FileExists(t, logFilePath) + text, err := os.ReadFile(logFilePath) + require.Nil(t, err) + require.Contains(t, string(text), "it works\n") + lines := strings.Split(string(text), "\n") + require.Equal(t, 101, len(lines)) +} + +func TestFileDriver_Find(t *testing.T) { + setupFileDriverTest() + t.Cleanup(cleanupFileDriverTest) + + d, err := newFileLogDriver(nil) + require.Nil(t, err) + defer d.Close() + + id := primitive.NewObjectID() + + batch := 1000 + var lines []string + for i := 0; i < 10; i++ { + for j := 0; j < batch; j++ { + line := fmt.Sprintf("line: %d", i*batch+j+1) + lines = append(lines, line) + } + err = d.WriteLines(id.Hex(), lines) + require.Nil(t, err) + lines = []string{} + } + + driver := d + + lines, err = driver.Find(id.Hex(), "", 0, 10) + require.Nil(t, err) + require.Equal(t, 10, len(lines)) + require.Equal(t, "line: 1", lines[0]) + require.Equal(t, "line: 10", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 0, 1) + require.Nil(t, err) + require.Equal(t, 1, len(lines)) + require.Equal(t, "line: 1", lines[0]) + require.Equal(t, "line: 1", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 0, 1000) + require.Nil(t, err) + require.Equal(t, 1000, len(lines)) + require.Equal(t, "line: 1", lines[0]) + require.Equal(t, "line: 1000", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 1000, 1000) + require.Nil(t, err) + require.Equal(t, 1000, len(lines)) + require.Equal(t, "line: 1001", lines[0]) + require.Equal(t, "line: 2000", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 1001, 1000) + require.Nil(t, err) + require.Equal(t, 1000, len(lines)) + require.Equal(t, "line: 1002", lines[0]) + require.Equal(t, "line: 2001", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 1001, 999) + require.Nil(t, err) + require.Equal(t, 999, len(lines)) + require.Equal(t, "line: 1002", lines[0]) + require.Equal(t, "line: 2000", lines[len(lines)-1]) + + lines, err = driver.Find(id.Hex(), "", 999, 2001) + require.Nil(t, err) + require.Equal(t, 2001, len(lines)) + require.Equal(t, "line: 1000", lines[0]) + require.Equal(t, "line: 3000", lines[len(lines)-1]) + + cleanupFileDriverTest() +} diff --git a/core/task/log/interface.go b/core/task/log/interface.go new file mode 100644 index 00000000..fd3745cb --- /dev/null +++ b/core/task/log/interface.go @@ -0,0 +1,10 @@ +package log + +type Driver interface { + Init() (err error) + Close() (err error) + WriteLine(id string, line string) (err error) + WriteLines(id string, lines []string) (err error) + Find(id string, pattern string, skip int, limit int) (lines []string, err error) + Count(id string, pattern string) (n int, err error) +} diff --git a/core/task/scheduler/options.go b/core/task/scheduler/options.go new file mode 100644 index 00000000..a6600fab --- /dev/null +++ b/core/task/scheduler/options.go @@ -0,0 +1,20 @@ +package scheduler + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "time" +) + +type Option func(svc interfaces.TaskSchedulerService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.TaskSchedulerService) { + svc.SetConfigPath(path) + } +} + +func WithInterval(interval time.Duration) Option { + return func(svc interfaces.TaskSchedulerService) { + svc.SetInterval(interval) + } +} diff --git a/core/task/scheduler/service.go b/core/task/scheduler/service.go new file mode 100644 index 00000000..fb858545 --- /dev/null +++ b/core/task/scheduler/service.go @@ -0,0 +1,264 @@ +package scheduler + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/task" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type Service struct { + // dependencies + interfaces.TaskBaseService + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + svr interfaces.GrpcServer + handlerSvc interfaces.TaskHandlerService + + // settings + interval time.Duration +} + +func (svc *Service) Start() { + go svc.initTaskStatus() + go svc.cleanupTasks() + svc.Wait() + svc.Stop() +} + +func (svc *Service) Enqueue(t interfaces.Task) (t2 interfaces.Task, err error) { + // set task status + t.SetStatus(constants.TaskStatusPending) + + // user + var u *models.User + if !t.GetUserId().IsZero() { + u, _ = svc.modelSvc.GetUserById(t.GetUserId()) + } + + // add task + if err = delegate.NewModelDelegate(t, u).Add(); err != nil { + return nil, err + } + + // task queue item + tq := &models.TaskQueueItem{ + Id: t.GetId(), + Priority: t.GetPriority(), + NodeId: t.GetNodeId(), + } + + // task stat + ts := &models.TaskStat{ + Id: t.GetId(), + CreateTs: time.Now(), + } + + // enqueue task + _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).Insert(tq) + if err != nil { + return nil, trace.TraceError(err) + } + + // add task stat + _, err = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).Insert(ts) + if err != nil { + return nil, trace.TraceError(err) + } + + // success + return t, nil +} + +func (svc *Service) Cancel(id primitive.ObjectID, args ...interface{}) (err error) { + // task + t, err := svc.modelSvc.GetTaskById(id) + if err != nil { + return trace.TraceError(err) + } + + // initial status + initialStatus := t.Status + + // set task status as "cancelled" + _ = svc.SaveTask(t, constants.TaskStatusCancelled) + + // set status of pending tasks as "cancelled" and remove from task item queue + if initialStatus == constants.TaskStatusPending { + // remove from task item queue + if err := mongo.GetMongoCol(interfaces.ModelColNameTaskQueue).DeleteId(t.GetId()); err != nil { + return trace.TraceError(err) + } + return nil + } + + // whether task is running on master node + isMasterTask, err := svc.isMasterNode(t) + if err != nil { + // when error, force status being set as "cancelled" + return svc.SaveTask(t, constants.TaskStatusCancelled) + } + + // node + n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) + if err != nil { + return trace.TraceError(err) + } + + if isMasterTask { + // cancel task on master + if err := svc.handlerSvc.Cancel(id); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } else { + // send to cancel task on worker nodes + if err := svc.svr.SendStreamMessageWithData("node:"+n.GetKey(), grpc.StreamMessageCode_CANCEL_TASK, t); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } +} + +func (svc *Service) SetInterval(interval time.Duration) { + svc.interval = interval +} + +// initTaskStatus initialize task status of existing tasks +func (svc *Service) initTaskStatus() { + // set status of running tasks as TaskStatusAbnormal + runningTasks, err := svc.modelSvc.GetTaskList(bson.M{ + "status": bson.M{ + "$in": []string{ + constants.TaskStatusPending, + constants.TaskStatusRunning, + }, + }, + }, nil) + if err != nil { + if err == mongo2.ErrNoDocuments { + return + } + trace.PrintError(err) + } + for _, t := range runningTasks { + go func(t *models.Task) { + if err := svc.SaveTask(t, constants.TaskStatusAbnormal); err != nil { + trace.PrintError(err) + } + }(&t) + } + if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskQueue).DeleteList(nil); err != nil { + return + } +} + +func (svc *Service) isMasterNode(t *models.Task) (ok bool, err error) { + if t.GetNodeId().IsZero() { + return false, trace.TraceError(errors.ErrorTaskNoNodeId) + } + n, err := svc.modelSvc.GetNodeById(t.GetNodeId()) + if err != nil { + if err == mongo2.ErrNoDocuments { + return false, trace.TraceError(errors.ErrorTaskNodeNotFound) + } + return false, trace.TraceError(err) + } + return n.IsMaster, nil +} + +func (svc *Service) cleanupTasks() { + for { + // task stats over 30 days ago + taskStats, err := svc.modelSvc.GetTaskStatList(bson.M{ + "create_ts": bson.M{ + "$lt": time.Now().Add(-30 * 24 * time.Hour), + }, + }, nil) + if err != nil { + time.Sleep(30 * time.Minute) + continue + } + + // task ids + var ids []primitive.ObjectID + for _, ts := range taskStats { + ids = append(ids, ts.Id) + } + + if len(ids) > 0 { + // remove tasks + if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTask).DeleteList(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + + // remove task stats + if err := svc.modelSvc.GetBaseService(interfaces.ModelIdTaskStat).DeleteList(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + } + + time.Sleep(30 * time.Minute) + } +} + +func NewTaskSchedulerService() (svc2 interfaces.TaskSchedulerService, err error) { + // base service + baseSvc, err := task.NewBaseService() + if err != nil { + return nil, trace.TraceError(err) + } + + // service + svc := &Service{ + TaskBaseService: baseSvc, + interval: 5 * time.Second, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + nodeCfgSvc interfaces.NodeConfigService, + modelSvc service.ModelService, + svr interfaces.GrpcServer, + handlerSvc interfaces.TaskHandlerService, + ) { + svc.nodeCfgSvc = nodeCfgSvc + svc.modelSvc = modelSvc + svc.svr = svr + svc.handlerSvc = handlerSvc + }); err != nil { + return nil, err + } + + return svc, nil +} + +var svc interfaces.TaskSchedulerService + +func GetTaskSchedulerService() (svr interfaces.TaskSchedulerService, err error) { + if svc != nil { + return svc, nil + } + svc, err = NewTaskSchedulerService() + if err != nil { + return nil, err + } + return svc, nil +} diff --git a/core/task/scheduler/service_v2.go b/core/task/scheduler/service_v2.go new file mode 100644 index 00000000..b5b6cfa4 --- /dev/null +++ b/core/task/scheduler/service_v2.go @@ -0,0 +1,264 @@ +package scheduler + +import ( + errors2 "errors" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + grpc "github.com/crawlab-team/crawlab/grpc" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type ServiceV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + svr interfaces.GrpcServer + handlerSvc interfaces.TaskHandlerService + + // settings + interval time.Duration +} + +func (svc *ServiceV2) Start() { + go svc.initTaskStatus() + go svc.cleanupTasks() + utils.DefaultWait() +} + +func (svc *ServiceV2) Enqueue(t *models.TaskV2, by primitive.ObjectID) (t2 *models.TaskV2, err error) { + // set task status + t.Status = constants.TaskStatusPending + t.SetCreatedBy(by) + t.SetUpdated(by) + + // add task + taskModelSvc := service.NewModelServiceV2[models.TaskV2]() + _, err = taskModelSvc.InsertOne(*t) + if err != nil { + return nil, err + } + + // task queue item + tq := models.TaskQueueItemV2{ + Priority: t.Priority, + NodeId: t.NodeId, + } + tq.SetId(t.Id) + + // task stat + ts := models.TaskStatV2{ + CreateTs: time.Now(), + } + ts.SetId(t.Id) + + // enqueue task + _, err = service.NewModelServiceV2[models.TaskQueueItemV2]().InsertOne(tq) + if err != nil { + return nil, trace.TraceError(err) + } + + // add task stat + _, err = service.NewModelServiceV2[models.TaskStatV2]().InsertOne(ts) + if err != nil { + return nil, trace.TraceError(err) + } + + // success + return t, nil +} + +func (svc *ServiceV2) Cancel(id primitive.ObjectID, by primitive.ObjectID) (err error) { + // task + t, err := service.NewModelServiceV2[models.TaskV2]().GetById(id) + if err != nil { + return trace.TraceError(err) + } + + // initial status + initialStatus := t.Status + + // set task status as "cancelled" + t.Status = constants.TaskStatusCancelled + _ = svc.SaveTask(t, by) + + // set status of pending tasks as "cancelled" and remove from task item queue + if initialStatus == constants.TaskStatusPending { + // remove from task item queue + if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteById(t.Id); err != nil { + return trace.TraceError(err) + } + return nil + } + + // whether task is running on master node + isMasterTask, err := svc.isMasterNode(t) + if err != nil { + // when error, force status being set as "cancelled" + t.Status = constants.TaskStatusCancelled + return svc.SaveTask(t, by) + } + + // node + n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + if err != nil { + return trace.TraceError(err) + } + + if isMasterTask { + // cancel task on master + if err := svc.handlerSvc.Cancel(id); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } else { + // send to cancel task on worker nodes + if err := svc.svr.SendStreamMessageWithData("node:"+n.Key, grpc.StreamMessageCode_CANCEL_TASK, t); err != nil { + return trace.TraceError(err) + } + // cancel success + return nil + } +} + +func (svc *ServiceV2) SetInterval(interval time.Duration) { + svc.interval = interval +} + +func (svc *ServiceV2) SaveTask(t *models.TaskV2, by primitive.ObjectID) (err error) { + if t.Id.IsZero() { + t.SetCreated(by) + t.SetUpdated(by) + _, err = service.NewModelServiceV2[models.TaskV2]().InsertOne(*t) + return err + } else { + t.SetUpdated(by) + return service.NewModelServiceV2[models.TaskV2]().ReplaceById(t.Id, *t) + } +} + +// initTaskStatus initialize task status of existing tasks +func (svc *ServiceV2) initTaskStatus() { + // set status of running tasks as TaskStatusAbnormal + runningTasks, err := service.NewModelServiceV2[models.TaskV2]().GetMany(bson.M{ + "status": bson.M{ + "$in": []string{ + constants.TaskStatusPending, + constants.TaskStatusRunning, + }, + }, + }, nil) + if err != nil { + if errors2.Is(err, mongo2.ErrNoDocuments) { + return + } + trace.PrintError(err) + } + for _, t := range runningTasks { + go func(t *models.TaskV2) { + t.Status = constants.TaskStatusAbnormal + if err := svc.SaveTask(t, primitive.NilObjectID); err != nil { + trace.PrintError(err) + } + }(&t) + } + if err := service.NewModelServiceV2[models.TaskQueueItemV2]().DeleteMany(nil); err != nil { + return + } +} + +func (svc *ServiceV2) isMasterNode(t *models.TaskV2) (ok bool, err error) { + if t.NodeId.IsZero() { + return false, trace.TraceError(errors.ErrorTaskNoNodeId) + } + n, err := service.NewModelServiceV2[models.NodeV2]().GetById(t.NodeId) + if err != nil { + if errors2.Is(err, mongo2.ErrNoDocuments) { + return false, trace.TraceError(errors.ErrorTaskNodeNotFound) + } + return false, trace.TraceError(err) + } + return n.IsMaster, nil +} + +func (svc *ServiceV2) cleanupTasks() { + for { + // task stats over 30 days ago + taskStats, err := service.NewModelServiceV2[models.TaskStatV2]().GetMany(bson.M{ + "create_ts": bson.M{ + "$lt": time.Now().Add(-30 * 24 * time.Hour), + }, + }, nil) + if err != nil { + time.Sleep(30 * time.Minute) + continue + } + + // task ids + var ids []primitive.ObjectID + for _, ts := range taskStats { + ids = append(ids, ts.Id) + } + + if len(ids) > 0 { + // remove tasks + if err := service.NewModelServiceV2[models.TaskV2]().DeleteMany(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + + // remove task stats + if err := service.NewModelServiceV2[models.TaskStatV2]().DeleteMany(bson.M{ + "_id": bson.M{"$in": ids}, + }); err != nil { + trace.PrintError(err) + } + } + + time.Sleep(30 * time.Minute) + } +} + +func NewTaskSchedulerServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + interval: 5 * time.Second, + } + + // dependency injection + if err := container.GetContainer().Invoke(func( + nodeCfgSvc interfaces.NodeConfigService, + svr interfaces.GrpcServer, + handlerSvc interfaces.TaskHandlerService, + ) { + svc.nodeCfgSvc = nodeCfgSvc + svc.svr = svr + svc.handlerSvc = handlerSvc + }); err != nil { + return nil, err + } + + return svc, nil +} + +var svcV2 *ServiceV2 + +func GetTaskSchedulerServiceV2() (svr *ServiceV2, err error) { + if svcV2 != nil { + return svcV2, nil + } + svcV2, err = NewTaskSchedulerServiceV2() + if err != nil { + return nil, err + } + return svcV2, nil +} diff --git a/core/task/stats/options.go b/core/task/stats/options.go new file mode 100644 index 00000000..37e7a823 --- /dev/null +++ b/core/task/stats/options.go @@ -0,0 +1,11 @@ +package stats + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(service interfaces.TaskStatsService) + +func WithConfigPath(path string) Option { + return func(svc interfaces.TaskStatsService) { + svc.SetConfigPath(path) + } +} diff --git a/core/task/stats/service.go b/core/task/stats/service.go new file mode 100644 index 00000000..ee12d762 --- /dev/null +++ b/core/task/stats/service.go @@ -0,0 +1,155 @@ +package stats + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/task" + "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type Service struct { + // dependencies + interfaces.TaskBaseService + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + + // internals + mu sync.Mutex + resultServices sync.Map + rsTtl time.Duration + logDriver log.Driver +} + +func (svc *Service) Init() (err error) { + go svc.cleanup() + return nil +} + +func (svc *Service) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { + resultSvc, err := svc.getResultService(id) + if err != nil { + return err + } + if err := resultSvc.Insert(records...); err != nil { + return err + } + go svc.updateTaskStats(id, len(records)) + return nil +} + +func (svc *Service) InsertLogs(id primitive.ObjectID, logs ...string) (err error) { + return svc.logDriver.WriteLines(id.Hex(), logs) +} + +func (svc *Service) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { + // atomic operation + svc.mu.Lock() + defer svc.mu.Unlock() + + // attempt to get from cache + res, _ := svc.resultServices.Load(id.Hex()) + if res != nil { + // hit in cache + resultSvc, ok := res.(interfaces.ResultService) + resultSvc.SetTime(time.Now()) + if ok { + return resultSvc, nil + } + } + + // task + t, err := svc.modelSvc.GetTaskById(id) + if err != nil { + return nil, err + } + + // result service + resultSvc, err = result.GetResultService(t.SpiderId) + if err != nil { + return nil, err + } + + // store in cache + svc.resultServices.Store(id.Hex(), resultSvc) + + return resultSvc, nil +} + +func (svc *Service) updateTaskStats(id primitive.ObjectID, resultCount int) { + _ = mongo.GetMongoCol(interfaces.ModelColNameTaskStat).UpdateId(id, bson.M{ + "$inc": bson.M{ + "result_count": resultCount, + }, + }) +} + +func (svc *Service) cleanup() { + for { + // atomic operation + svc.mu.Lock() + + svc.resultServices.Range(func(key, value interface{}) bool { + rs := value.(interfaces.ResultService) + if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { + svc.resultServices.Delete(key) + } + return true + }) + + svc.mu.Unlock() + + time.Sleep(10 * time.Minute) + } +} + +func NewTaskStatsService() (svc2 interfaces.TaskStatsService, err error) { + // base service + baseSvc, err := task.NewBaseService() + if err != nil { + return nil, trace.TraceError(err) + } + + // service + svc := &Service{ + mu: sync.Mutex{}, + TaskBaseService: baseSvc, + resultServices: sync.Map{}, + } + + // dependency injection + if err := container.GetContainer().Invoke(func(nodeCfgSvc interfaces.NodeConfigService, modelSvc service.ModelService) { + svc.nodeCfgSvc = nodeCfgSvc + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // log driver + svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) + if err != nil { + return nil, err + } + + return svc, nil +} + +var _service interfaces.TaskStatsService + +func GetTaskStatsService() (svr interfaces.TaskStatsService, err error) { + if _service != nil { + return _service, nil + } + _service, err = NewTaskStatsService() + if err != nil { + return nil, err + } + return _service, nil +} diff --git a/core/task/stats/service_v2.go b/core/task/stats/service_v2.go new file mode 100644 index 00000000..b198c95d --- /dev/null +++ b/core/task/stats/service_v2.go @@ -0,0 +1,143 @@ +package stats + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + nodeconfig "github.com/crawlab-team/crawlab/core/node/config" + "github.com/crawlab-team/crawlab/core/result" + "github.com/crawlab-team/crawlab/core/task/log" + "github.com/crawlab-team/go-trace" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "sync" + "time" +) + +type ServiceV2 struct { + // dependencies + nodeCfgSvc interfaces.NodeConfigService + modelSvc service.ModelService + + // internals + mu sync.Mutex + resultServices sync.Map + rsTtl time.Duration + logDriver log.Driver +} + +func (svc *ServiceV2) Init() (err error) { + go svc.cleanup() + return nil +} + +func (svc *ServiceV2) InsertData(id primitive.ObjectID, records ...interface{}) (err error) { + resultSvc, err := svc.getResultService(id) + if err != nil { + return err + } + if err := resultSvc.Insert(records...); err != nil { + return err + } + go svc.updateTaskStats(id, len(records)) + return nil +} + +func (svc *ServiceV2) InsertLogs(id primitive.ObjectID, logs ...string) (err error) { + return svc.logDriver.WriteLines(id.Hex(), logs) +} + +func (svc *ServiceV2) getResultService(id primitive.ObjectID) (resultSvc interfaces.ResultService, err error) { + // atomic operation + svc.mu.Lock() + defer svc.mu.Unlock() + + // attempt to get from cache + res, _ := svc.resultServices.Load(id.Hex()) + if res != nil { + // hit in cache + resultSvc, ok := res.(interfaces.ResultService) + resultSvc.SetTime(time.Now()) + if ok { + return resultSvc, nil + } + } + + // task + t, err := svc.modelSvc.GetTaskById(id) + if err != nil { + return nil, err + } + + // result service + resultSvc, err = result.GetResultService(t.SpiderId) + if err != nil { + return nil, err + } + + // store in cache + svc.resultServices.Store(id.Hex(), resultSvc) + + return resultSvc, nil +} + +func (svc *ServiceV2) updateTaskStats(id primitive.ObjectID, resultCount int) { + err := service.NewModelServiceV2[models.TaskStatV2]().UpdateById(id, bson.M{ + "$inc": bson.M{ + "result_count": resultCount, + }, + }) + if err != nil { + trace.PrintError(err) + } +} + +func (svc *ServiceV2) cleanup() { + for { + // atomic operation + svc.mu.Lock() + + svc.resultServices.Range(func(key, value interface{}) bool { + rs := value.(interfaces.ResultService) + if time.Now().After(rs.GetTime().Add(svc.rsTtl)) { + svc.resultServices.Delete(key) + } + return true + }) + + svc.mu.Unlock() + + time.Sleep(10 * time.Minute) + } +} + +func NewTaskStatsServiceV2() (svc2 *ServiceV2, err error) { + // service + svc := &ServiceV2{ + mu: sync.Mutex{}, + resultServices: sync.Map{}, + } + + svc.nodeCfgSvc = nodeconfig.GetNodeConfigService() + + // log driver + svc.logDriver, err = log.GetLogDriver(log.DriverTypeFile) + if err != nil { + return nil, err + } + + return svc, nil +} + +var _serviceV2 *ServiceV2 + +func GetTaskStatsServiceV2() (svr *ServiceV2, err error) { + if _serviceV2 != nil { + return _serviceV2, nil + } + _serviceV2, err = NewTaskStatsServiceV2() + if err != nil { + return nil, err + } + return _serviceV2, nil +} diff --git a/core/user/options.go b/core/user/options.go new file mode 100644 index 00000000..81c208d7 --- /dev/null +++ b/core/user/options.go @@ -0,0 +1,11 @@ +package user + +import "github.com/crawlab-team/crawlab/core/interfaces" + +type Option func(svc interfaces.UserService) + +func WithJwtSecret(secret string) Option { + return func(svc interfaces.UserService) { + svc.SetJwtSecret(secret) + } +} diff --git a/core/user/service.go b/core/user/service.go new file mode 100644 index 00000000..97a2ad9d --- /dev/null +++ b/core/user/service.go @@ -0,0 +1,238 @@ +package user + +import ( + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/container" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/delegate" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type Service struct { + // settings variables + jwtSecret string + jwtSigningMethod jwt.SigningMethod + + // dependencies + modelSvc service.ModelService +} + +func (svc *Service) Init() (err error) { + _, err = svc.modelSvc.GetUserByUsername(constants.DefaultAdminUsername, nil) + if err == nil { + return nil + } + if err.Error() != mongo.ErrNoDocuments.Error() { + return err + } + return svc.Create(&interfaces.UserCreateOptions{ + Username: constants.DefaultAdminUsername, + Password: constants.DefaultAdminPassword, + Role: constants.RoleAdmin, + }) +} + +func (svc *Service) SetJwtSecret(secret string) { + svc.jwtSecret = secret +} + +func (svc *Service) SetJwtSigningMethod(method jwt.SigningMethod) { + svc.jwtSigningMethod = method +} + +func (svc *Service) Create(opts *interfaces.UserCreateOptions, args ...interface{}) (err error) { + actor := utils.GetUserFromArgs(args...) + + // validate options + if opts.Username == "" || opts.Password == "" { + return trace.TraceError(errors.ErrorUserMissingRequiredFields) + } + if len(opts.Password) < 5 { + return trace.TraceError(errors.ErrorUserInvalidPassword) + } + + // normalize options + if opts.Role == "" { + opts.Role = constants.RoleNormal + } + + // check if user exists + if u, err := svc.modelSvc.GetUserByUsername(opts.Username, nil); err == nil && u != nil && !u.Id.IsZero() { + return trace.TraceError(errors.ErrorUserAlreadyExists) + } + + // transaction + return mongo2.RunTransaction(func(ctx mongo.SessionContext) error { + // add user + u := &models.User{ + Username: opts.Username, + Role: opts.Role, + Email: opts.Email, + } + if err := delegate.NewModelDelegate(u, actor).Add(); err != nil { + return err + } + + // add password + p := &models.Password{ + Id: u.Id, + Password: utils.EncryptMd5(opts.Password), + } + if err := delegate.NewModelDelegate(p, actor).Add(); err != nil { + return err + } + + return nil + }) +} + +func (svc *Service) Login(opts *interfaces.UserLoginOptions) (token string, u interfaces.User, err error) { + u, err = svc.modelSvc.GetUserByUsername(opts.Username, nil) + if err != nil { + return "", nil, err + } + p, err := svc.modelSvc.GetPasswordById(u.GetId()) + if err != nil { + return "", nil, err + } + if p.Password != utils.EncryptMd5(opts.Password) { + return "", nil, errors.ErrorUserMismatch + } + token, err = svc.makeToken(u) + if err != nil { + return "", nil, err + } + return token, u, nil +} + +func (svc *Service) CheckToken(tokenStr string) (u interfaces.User, err error) { + return svc.checkToken(tokenStr) +} + +func (svc *Service) ChangePassword(id primitive.ObjectID, password string, args ...interface{}) (err error) { + actor := utils.GetUserFromArgs(args...) + + p, err := svc.modelSvc.GetPasswordById(id) + if err != nil { + return err + } + p.Password = utils.EncryptMd5(password) + if err := delegate.NewModelDelegate(p, actor).Save(); err != nil { + return err + } + return nil +} + +func (svc *Service) MakeToken(user interfaces.User) (tokenStr string, err error) { + return svc.makeToken(user) +} + +func (svc *Service) GetCurrentUser(c *gin.Context) (user interfaces.User, err error) { + // token string + tokenStr := c.GetHeader("Authorization") + + // user + u, err := userSvc.CheckToken(tokenStr) + if err != nil { + return nil, err + } + + return u, nil +} + +func (svc *Service) makeToken(user interfaces.User) (tokenStr string, err error) { + token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ + "id": user.GetId(), + "username": user.GetUsername(), + "nbf": time.Now().Unix(), + }) + return token.SignedString([]byte(svc.jwtSecret)) +} + +func (svc *Service) checkToken(tokenStr string) (user interfaces.User, err error) { + token, err := jwt.Parse(tokenStr, svc.getSecretFunc()) + if err != nil { + return + } + + claim, ok := token.Claims.(jwt.MapClaims) + if !ok { + err = errors.ErrorUserInvalidType + return + } + + if !token.Valid { + err = errors.ErrorUserInvalidToken + return + } + + id, err := primitive.ObjectIDFromHex(claim["id"].(string)) + if err != nil { + return user, err + } + username := claim["username"].(string) + user, err = svc.modelSvc.GetUserById(id) + if err != nil { + err = errors.ErrorUserNotExists + return + } + + if username != user.GetUsername() { + err = errors.ErrorUserMismatch + return + } + + return +} + +func (svc *Service) getSecretFunc() jwt.Keyfunc { + return func(token *jwt.Token) (interface{}, error) { + return []byte(svc.jwtSecret), nil + } +} + +func NewUserService() (svc2 interfaces.UserService, err error) { + // service + svc := &Service{ + jwtSecret: "crawlab", + jwtSigningMethod: jwt.SigningMethodHS256, + } + + // dependency injection + if err := container.GetContainer().Invoke(func(modelSvc service.ModelService) { + svc.modelSvc = modelSvc + }); err != nil { + return nil, trace.TraceError(err) + } + + // initialize + if err := svc.Init(); err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} + +var userSvc interfaces.UserService + +func GetUserService() (svc interfaces.UserService, err error) { + if userSvc != nil { + return userSvc, nil + } + svc, err = NewUserService() + if err != nil { + return nil, err + } + userSvc = svc + return svc, nil +} diff --git a/core/user/service_v2.go b/core/user/service_v2.go new file mode 100644 index 00000000..e4aa72b6 --- /dev/null +++ b/core/user/service_v2.go @@ -0,0 +1,212 @@ +package user + +import ( + mongo2 "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "go.mongodb.org/mongo-driver/mongo" + "time" +) + +type ServiceV2 struct { + jwtSecret string + jwtSigningMethod jwt.SigningMethod + modelSvc *service.ModelServiceV2[models.UserV2] +} + +func (svc *ServiceV2) Init() (err error) { + _, err = svc.modelSvc.GetOne(bson.M{"username": constants.DefaultAdminUsername}, nil) + if err == nil { + return nil + } + if err.Error() != mongo.ErrNoDocuments.Error() { + return err + } + return svc.Create( + constants.DefaultAdminUsername, + constants.DefaultAdminPassword, + constants.RoleAdmin, + "", + primitive.NilObjectID, + ) +} + +func (svc *ServiceV2) SetJwtSecret(secret string) { + svc.jwtSecret = secret +} + +func (svc *ServiceV2) SetJwtSigningMethod(method jwt.SigningMethod) { + svc.jwtSigningMethod = method +} + +func (svc *ServiceV2) Create(username, password, role, email string, by primitive.ObjectID) (err error) { + // validate options + if username == "" || password == "" { + return trace.TraceError(errors.ErrorUserMissingRequiredFields) + } + if len(password) < 5 { + return trace.TraceError(errors.ErrorUserInvalidPassword) + } + + // normalize options + if role == "" { + role = constants.RoleNormal + } + + // check if user exists + if u, err := svc.modelSvc.GetOne(bson.M{"username": username}, nil); err == nil && u != nil && !u.Id.IsZero() { + return trace.TraceError(errors.ErrorUserAlreadyExists) + } + + // transaction + return mongo2.RunTransaction(func(ctx mongo.SessionContext) error { + // add user + u := models.UserV2{ + Username: username, + Role: role, + Password: utils.EncryptMd5(password), + Email: email, + } + u.SetCreated(by) + u.SetUpdated(by) + _, err = svc.modelSvc.InsertOne(u) + + return err + }) +} + +func (svc *ServiceV2) Login(username, password string) (token string, u *models.UserV2, err error) { + u, err = svc.modelSvc.GetOne(bson.M{"username": username}, nil) + if err != nil { + return "", nil, err + } + if u.Password != utils.EncryptMd5(password) { + return "", nil, errors.ErrorUserMismatch + } + token, err = svc.makeToken(u) + if err != nil { + return "", nil, err + } + return token, u, nil +} + +func (svc *ServiceV2) CheckToken(tokenStr string) (u *models.UserV2, err error) { + return svc.checkToken(tokenStr) +} + +func (svc *ServiceV2) ChangePassword(id primitive.ObjectID, password string, by primitive.ObjectID) (err error) { + u, err := svc.modelSvc.GetById(id) + if err != nil { + return err + } + u.Password = utils.EncryptMd5(password) + u.SetCreatedBy(by) + return svc.modelSvc.ReplaceById(id, *u) +} + +func (svc *ServiceV2) MakeToken(user *models.UserV2) (tokenStr string, err error) { + return svc.makeToken(user) +} + +func (svc *ServiceV2) GetCurrentUser(c *gin.Context) (user interfaces.User, err error) { + // token string + tokenStr := c.GetHeader("Authorization") + + // user + u, err := userSvc.CheckToken(tokenStr) + if err != nil { + return nil, err + } + + return u, nil +} + +func (svc *ServiceV2) makeToken(user *models.UserV2) (tokenStr string, err error) { + token := jwt.NewWithClaims(svc.jwtSigningMethod, jwt.MapClaims{ + "id": user.Id, + "username": user.Username, + "nbf": time.Now().Unix(), + }) + return token.SignedString([]byte(svc.jwtSecret)) +} + +func (svc *ServiceV2) checkToken(tokenStr string) (user *models.UserV2, err error) { + token, err := jwt.Parse(tokenStr, svc.getSecretFunc()) + if err != nil { + return + } + + claim, ok := token.Claims.(jwt.MapClaims) + if !ok { + err = errors.ErrorUserInvalidType + return + } + + if !token.Valid { + err = errors.ErrorUserInvalidToken + return + } + + id, err := primitive.ObjectIDFromHex(claim["id"].(string)) + if err != nil { + return user, err + } + username := claim["username"].(string) + user, err = svc.modelSvc.GetById(id) + if err != nil { + err = errors.ErrorUserNotExists + return + } + + if username != user.Username { + err = errors.ErrorUserMismatch + return + } + + return +} + +func (svc *ServiceV2) getSecretFunc() jwt.Keyfunc { + return func(token *jwt.Token) (interface{}, error) { + return []byte(svc.jwtSecret), nil + } +} + +func NewUserServiceV2() (svc *ServiceV2, err error) { + // service + svc = &ServiceV2{ + modelSvc: service.NewModelServiceV2[models.UserV2](), + jwtSecret: "crawlab", + jwtSigningMethod: jwt.SigningMethodHS256, + } + + // initialize + if err := svc.Init(); err != nil { + return nil, trace.TraceError(err) + } + + return svc, nil +} + +var userSvcV2 *ServiceV2 + +func GetUserServiceV2() (svc *ServiceV2, err error) { + if userSvcV2 != nil { + return userSvcV2, nil + } + svc, err = NewUserServiceV2() + if err != nil { + return nil, err + } + userSvcV2 = svc + return svc, nil +} diff --git a/core/user/test/base.go b/core/user/test/base.go new file mode 100644 index 00000000..392a7e06 --- /dev/null +++ b/core/user/test/base.go @@ -0,0 +1,65 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/service" + "github.com/crawlab-team/crawlab/core/user" + "go.uber.org/dig" + "testing" +) + +func init() { + var err error + T, err = NewTest() + if err != nil { + panic(err) + } +} + +var T *Test + +type Test struct { + // dependencies + modelSvc service.ModelService + userSvc interfaces.UserService + + // test data + TestUsername string + TestPassword string + TestNewPassword string +} + +func (t *Test) Setup(t2 *testing.T) { + var err error + t.userSvc, err = user.NewUserService() + if err != nil { + panic(err) + } + t2.Cleanup(t.Cleanup) +} + +func (t *Test) Cleanup() { + _ = t.modelSvc.GetBaseService(interfaces.ModelIdUser).DeleteList(nil) +} + +func NewTest() (t *Test, err error) { + // test + t = &Test{ + TestUsername: "test_username", + TestPassword: "test_password", + TestNewPassword: "test_new_password", + } + + // dependency injection + c := dig.New() + if err := c.Provide(service.GetService); err != nil { + return nil, err + } + if err := c.Invoke(func(modelSvc service.ModelService) { + t.modelSvc = modelSvc + }); err != nil { + return nil, err + } + + return t, nil +} diff --git a/core/user/test/user_service_test.go b/core/user/test/user_service_test.go new file mode 100644 index 00000000..0806c31e --- /dev/null +++ b/core/user/test/user_service_test.go @@ -0,0 +1,61 @@ +package test + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/utils" + "github.com/stretchr/testify/require" + "testing" +) + +func TestUserService_Init(t *testing.T) { + var err error + T.Setup(t) + + u, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) + require.Nil(t, err) + require.Equal(t, constants.DefaultAdminUsername, u.Username) + require.Equal(t, utils.EncryptMd5(constants.DefaultAdminPassword), u.Password) +} + +func TestUserService_Create_Login_CheckToken(t *testing.T) { + var err error + T.Setup(t) + + err = T.userSvc.Create(&interfaces.UserCreateOptions{ + Username: T.TestUsername, + Password: T.TestPassword, + }) + require.Nil(t, err) + + u, err := T.modelSvc.GetUserByUsernameWithPassword(T.TestUsername, nil) + require.Nil(t, err) + require.Equal(t, T.TestUsername, u.Username) + require.Equal(t, utils.EncryptMd5(T.TestPassword), u.Password) + + token, u2, err := T.userSvc.Login(&interfaces.UserLoginOptions{ + Username: T.TestUsername, + Password: T.TestPassword, + }) + require.Nil(t, err) + require.Greater(t, len(token), 10) + require.Equal(t, u.Username, u2.GetUsername()) + + u3, err := T.userSvc.CheckToken(token) + require.Nil(t, err) + require.Equal(t, u.Username, u3.GetUsername()) +} + +func TestUserService_ChangePassword(t *testing.T) { + var err error + T.Setup(t) + + u, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) + require.Nil(t, err) + err = T.userSvc.ChangePassword(u.Id, T.TestNewPassword) + require.Nil(t, err) + + u2, err := T.modelSvc.GetUserByUsernameWithPassword(constants.DefaultAdminUsername, nil) + require.Nil(t, err) + require.Equal(t, utils.EncryptMd5(T.TestNewPassword), u2.Password) +} diff --git a/core/utils/args.go b/core/utils/args.go new file mode 100644 index 00000000..e83f540a --- /dev/null +++ b/core/utils/args.go @@ -0,0 +1,17 @@ +package utils + +import "github.com/crawlab-team/crawlab/core/interfaces" + +func GetUserFromArgs(args ...interface{}) (u interfaces.User) { + for _, arg := range args { + switch arg.(type) { + case interfaces.User: + var ok bool + u, ok = arg.(interfaces.User) + if ok { + return u + } + } + } + return nil +} diff --git a/core/utils/array.go b/core/utils/array.go new file mode 100644 index 00000000..a5e958c6 --- /dev/null +++ b/core/utils/array.go @@ -0,0 +1,46 @@ +package utils + +import ( + "errors" + "math/rand" + "reflect" + "time" +) + +func StringArrayContains(arr []string, str string) bool { + for _, s := range arr { + if s == str { + return true + } + } + return false +} + +func GetArrayItems(array interface{}) (res []interface{}, err error) { + switch reflect.TypeOf(array).Kind() { + case reflect.Slice, reflect.Array: + s := reflect.ValueOf(array) + for i := 0; i < s.Len(); i++ { + obj, ok := s.Index(i).Interface().(interface{}) + if !ok { + return nil, errors.New("invalid type") + } + res = append(res, obj) + } + default: + return nil, errors.New("invalid type") + } + return res, nil +} + +func ShuffleArray(slice []interface{}) (err error) { + r := rand.New(rand.NewSource(time.Now().Unix())) + for len(slice) > 0 { + n := len(slice) + randIndex := r.Intn(n) + slice[n-1], slice[randIndex] = slice[randIndex], slice[n-1] + slice = slice[:n-1] + } + + return nil +} diff --git a/core/utils/backoff.go b/core/utils/backoff.go new file mode 100644 index 00000000..53658a8d --- /dev/null +++ b/core/utils/backoff.go @@ -0,0 +1,15 @@ +package utils + +import ( + "github.com/apex/log" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/go-trace" + "time" +) + +func BackoffErrorNotify(prefix string) backoff.Notify { + return func(err error, duration time.Duration) { + log.Errorf("%s error: %v. reattempt in %.1f seconds...", prefix, err, duration.Seconds()) + trace.PrintError(err) + } +} diff --git a/core/utils/binders/binder_col_name.go b/core/utils/binders/binder_col_name.go new file mode 100644 index 00000000..9e76224d --- /dev/null +++ b/core/utils/binders/binder_col_name.go @@ -0,0 +1,99 @@ +package binders + +import ( + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +func NewColNameBinder(id interfaces.ModelId) (b *ColNameBinder) { + return &ColNameBinder{id: id} +} + +type ColNameBinder struct { + id interfaces.ModelId +} + +func (b *ColNameBinder) Bind() (res interface{}, err error) { + switch b.id { + // system models + case interfaces.ModelIdArtifact: + return interfaces.ModelColNameArtifact, nil + case interfaces.ModelIdTag: + return interfaces.ModelColNameTag, nil + + // operation models + case interfaces.ModelIdNode: + return interfaces.ModelColNameNode, nil + case interfaces.ModelIdProject: + return interfaces.ModelColNameProject, nil + case interfaces.ModelIdSpider: + return interfaces.ModelColNameSpider, nil + case interfaces.ModelIdTask: + return interfaces.ModelColNameTask, nil + case interfaces.ModelIdJob: + return interfaces.ModelColNameJob, nil + case interfaces.ModelIdSchedule: + return interfaces.ModelColNameSchedule, nil + case interfaces.ModelIdUser: + return interfaces.ModelColNameUser, nil + case interfaces.ModelIdSetting: + return interfaces.ModelColNameSetting, nil + case interfaces.ModelIdToken: + return interfaces.ModelColNameToken, nil + case interfaces.ModelIdVariable: + return interfaces.ModelColNameVariable, nil + case interfaces.ModelIdTaskQueue: + return interfaces.ModelColNameTaskQueue, nil + case interfaces.ModelIdTaskStat: + return interfaces.ModelColNameTaskStat, nil + case interfaces.ModelIdSpiderStat: + return interfaces.ModelColNameSpiderStat, nil + case interfaces.ModelIdDataSource: + return interfaces.ModelColNameDataSource, nil + case interfaces.ModelIdDataCollection: + return interfaces.ModelColNameDataCollection, nil + case interfaces.ModelIdPassword: + return interfaces.ModelColNamePasswords, nil + case interfaces.ModelIdExtraValue: + return interfaces.ModelColNameExtraValues, nil + case interfaces.ModelIdGit: + return interfaces.ModelColNameGit, nil + case interfaces.ModelIdRole: + return interfaces.ModelColNameRole, nil + case interfaces.ModelIdUserRole: + return interfaces.ModelColNameUserRole, nil + case interfaces.ModelIdPermission: + return interfaces.ModelColNamePermission, nil + case interfaces.ModelIdRolePermission: + return interfaces.ModelColNameRolePermission, nil + case interfaces.ModelIdEnvironment: + return interfaces.ModelColNameEnvironment, nil + case interfaces.ModelIdDependencySetting: + return interfaces.ModelColNameDependencySetting, nil + + // invalid + default: + return res, errors.ErrorModelNotImplemented + } +} + +func (b *ColNameBinder) MustBind() (res interface{}) { + res, err := b.Bind() + if err != nil { + panic(err) + } + return res +} + +func (b *ColNameBinder) BindString() (res string, err error) { + res_, err := b.Bind() + if err != nil { + return "", err + } + res = res_.(string) + return res, nil +} + +func (b *ColNameBinder) MustBindString() (res string) { + return b.MustBind().(string) +} diff --git a/core/utils/bool.go b/core/utils/bool.go new file mode 100644 index 00000000..8d569683 --- /dev/null +++ b/core/utils/bool.go @@ -0,0 +1,12 @@ +package utils + +import "github.com/spf13/viper" + +func EnvIsTrue(key string, defaultOk bool) bool { + isTrueBool := viper.GetBool(key) + isTrueString := viper.GetString(key) + if isTrueString == "" { + return defaultOk + } + return isTrueBool || isTrueString == "Y" +} diff --git a/core/utils/bson.go b/core/utils/bson.go new file mode 100644 index 00000000..e6443e88 --- /dev/null +++ b/core/utils/bson.go @@ -0,0 +1,125 @@ +package utils + +import ( + "github.com/emirpasic/gods/sets/hashset" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" +) + +func BsonMEqual(v1, v2 bson.M) (ok bool) { + //ok = reflect.DeepEqual(v1, v2) + ok = bsonMEqual(v1, v2) + return ok +} + +func bsonMEqual(v1, v2 bson.M) (ok bool) { + // all keys + allKeys := hashset.New() + for key := range v1 { + allKeys.Add(key) + } + for key := range v2 { + allKeys.Add(key) + } + + for _, keyRes := range allKeys.Values() { + key := keyRes.(string) + v1Value, ok := v1[key] + if !ok { + return false + } + v2Value, ok := v2[key] + if !ok { + return false + } + + mode := 0 + + var v1ValueBsonM bson.M + var v1ValueBsonA bson.A + switch v1Value.(type) { + case bson.M: + mode = 1 + v1ValueBsonM = v1Value.(bson.M) + case bson.A: + mode = 2 + v1ValueBsonA = v1Value.(bson.A) + } + + var v2ValueBsonM bson.M + var v2ValueBsonA bson.A + switch v2Value.(type) { + case bson.M: + if mode != 1 { + return false + } + v2ValueBsonM = v2Value.(bson.M) + case bson.A: + if mode != 2 { + return false + } + v2ValueBsonA = v2Value.(bson.A) + } + + switch mode { + case 0: + if v1Value != v2Value { + return false + } + case 1: + if !bsonMEqual(v1ValueBsonM, v2ValueBsonM) { + return false + } + case 2: + if !reflect.DeepEqual(v1ValueBsonA, v2ValueBsonA) { + return false + } + default: + // not reachable + return false + } + } + + return true +} + +func NormalizeBsonMObjectId(m bson.M) (res bson.M) { + for k, v := range m { + switch v.(type) { + case string: + oid, err := primitive.ObjectIDFromHex(v.(string)) + if err == nil { + m[k] = oid + } + case bson.M: + m[k] = NormalizeBsonMObjectId(v.(bson.M)) + } + } + return m +} + +func DenormalizeBsonMObjectId(m bson.M) (res bson.M) { + for k, v := range m { + switch v.(type) { + case primitive.ObjectID: + m[k] = v.(primitive.ObjectID).Hex() + case bson.M: + m[k] = NormalizeBsonMObjectId(v.(bson.M)) + } + } + return m +} + +func NormalizeObjectId(v interface{}) (res interface{}) { + switch v.(type) { + case string: + oid, err := primitive.ObjectIDFromHex(v.(string)) + if err != nil { + return v + } + return oid + default: + return v + } +} diff --git a/core/utils/cache.go b/core/utils/cache.go new file mode 100644 index 00000000..b875b316 --- /dev/null +++ b/core/utils/cache.go @@ -0,0 +1,57 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +func GetFromDbCache(key string, getFn func() (string, error)) (res string, err error) { + col := mongo.GetMongoCol(constants.CacheColName) + + var d bson.M + if err := col.Find(bson.M{ + constants.CacheColKey: key, + }, nil).One(&d); err != nil { + if err != mongo2.ErrNoDocuments { + return "", err + } + + // get cache value + res, err = getFn() + if err != nil { + return "", err + } + + // save cache + d = bson.M{ + constants.CacheColKey: key, + constants.CacheColValue: res, + constants.CacheColTime: time.Now(), + } + if _, err := col.Insert(d); err != nil { + return "", err + } + return res, nil + } + + // type conversion + r, ok := d[constants.CacheColValue] + if !ok { + if err := col.Delete(bson.M{constants.CacheColKey: key}); err != nil { + return "", err + } + return GetFromDbCache(key, getFn) + } + res, ok = r.(string) + if !ok { + if err := col.Delete(bson.M{constants.CacheColKey: key}); err != nil { + return "", err + } + return GetFromDbCache(key, getFn) + } + + return res, nil +} diff --git a/core/utils/chan.go b/core/utils/chan.go new file mode 100644 index 00000000..c0144340 --- /dev/null +++ b/core/utils/chan.go @@ -0,0 +1,40 @@ +package utils + +import ( + "sync" +) + +var TaskExecChanMap = NewChanMap() + +type ChanMap struct { + m sync.Map +} + +func NewChanMap() *ChanMap { + return &ChanMap{m: sync.Map{}} +} + +func (cm *ChanMap) Chan(key string) chan string { + if ch, ok := cm.m.Load(key); ok { + return ch.(interface{}).(chan string) + } + ch := make(chan string, 10) + cm.m.Store(key, ch) + return ch +} + +func (cm *ChanMap) ChanBlocked(key string) chan string { + if ch, ok := cm.m.Load(key); ok { + return ch.(interface{}).(chan string) + } + ch := make(chan string) + cm.m.Store(key, ch) + return ch +} + +func (cm *ChanMap) HasChanKey(key string) bool { + if _, ok := cm.m.Load(key); ok { + return true + } + return false +} diff --git a/core/utils/chan_test.go b/core/utils/chan_test.go new file mode 100644 index 00000000..4bc75917 --- /dev/null +++ b/core/utils/chan_test.go @@ -0,0 +1,78 @@ +package utils + +import ( + . "github.com/smartystreets/goconvey/convey" + "sync" + "testing" +) + +func TestNewChanMap(t *testing.T) { + mapTest := sync.Map{} + chanTest := make(chan string) + test := "test" + + Convey("Call NewChanMap to generate ChanMap", t, func() { + mapTest.Store("test", chanTest) + chanMapTest := ChanMap{mapTest} + chanMap := NewChanMap() + chanMap.m.Store("test", chanTest) + + Convey(test, func() { + v1, ok := chanMap.m.Load("test") + So(ok, ShouldBeTrue) + v2, ok := chanMapTest.m.Load("test") + So(ok, ShouldBeTrue) + So(v1, ShouldResemble, v2) + }) + }) +} + +func TestChan(t *testing.T) { + mapTest := sync.Map{} + chanTest := make(chan string) + mapTest.Store("test", chanTest) + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.Chan("test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.Chan("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + v, ok := chanMapTest.m.Load("test2") + So(ok, ShouldBeTrue) + So(v, ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(10, ShouldEqual, cap(ch2)) + }) + }) +} + +func TestChanBlocked(t *testing.T) { + mapTest := sync.Map{} + chanTest := make(chan string) + mapTest.Store("test", chanTest) + chanMapTest := ChanMap{mapTest} + + Convey("Test Chan use exist key", t, func() { + ch1 := chanMapTest.ChanBlocked("test") + Convey("ch1 should equal chanTest", func() { + So(ch1, ShouldEqual, chanTest) + }) + }) + Convey("Test Chan use no-exist key", t, func() { + ch2 := chanMapTest.ChanBlocked("test2") + Convey("ch2 should equal chanMapTest.m[test2]", func() { + v, ok := chanMapTest.m.Load("test2") + So(ok, ShouldBeTrue) + So(v, ShouldEqual, ch2) + }) + Convey("Cap of chanMapTest.m[test2] should equal 10", func() { + So(0, ShouldEqual, cap(ch2)) + }) + }) +} diff --git a/core/utils/cockroachdb.go b/core/utils/cockroachdb.go new file mode 100644 index 00000000..650c32f1 --- /dev/null +++ b/core/utils/cockroachdb.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mssql" + "time" +) + +func GetCockroachdbSession(ds *models.DataSource) (s db.Session, err error) { + return getCockroachdbSession(context.Background(), ds) +} + +func GetCockroachdbSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getCockroachdbSession(ctx, ds) +} + +func getCockroachdbSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultCockroachdbPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/cron.go b/core/utils/cron.go new file mode 100644 index 00000000..e8838d2f --- /dev/null +++ b/core/utils/cron.go @@ -0,0 +1,176 @@ +package utils + +import ( + "fmt" + "math" + "strconv" + "strings" +) + +// cronBounds provides a range of acceptable values (plus a map of name to value). +type cronBounds struct { + min, max uint + names map[string]uint +} + +type cronUtils struct { + // The cronBounds for each field. + seconds cronBounds + minutes cronBounds + hours cronBounds + dom cronBounds + months cronBounds + dow cronBounds + + // Set the top bit if a star was included in the expression. + starBit uint64 +} + +// getRange returns the bits indicated by the given expression: +// number | number "-" number [ "/" number ] +// or error parsing range. +func (u *cronUtils) getRange(expr string, r cronBounds) (uint64, error) { + var ( + start, end, step uint + rangeAndStep = strings.Split(expr, "/") + lowAndHigh = strings.Split(rangeAndStep[0], "-") + singleDigit = len(lowAndHigh) == 1 + err error + ) + + var extra uint64 + if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" { + start = r.min + end = r.max + extra = CronUtils.starBit + } else { + start, err = u.parseIntOrName(lowAndHigh[0], r.names) + if err != nil { + return 0, err + } + switch len(lowAndHigh) { + case 1: + end = start + case 2: + end, err = u.parseIntOrName(lowAndHigh[1], r.names) + if err != nil { + return 0, err + } + default: + return 0, fmt.Errorf("too many hyphens: %s", expr) + } + } + + switch len(rangeAndStep) { + case 1: + step = 1 + case 2: + step, err = u.mustParseInt(rangeAndStep[1]) + if err != nil { + return 0, err + } + + // Special handling: "N/step" means "N-max/step". + if singleDigit { + end = r.max + } + if step > 1 { + extra = 0 + } + default: + return 0, fmt.Errorf("too many slashes: %s", expr) + } + + if start < r.min { + return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr) + } + if end > r.max { + return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr) + } + if start > end { + return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr) + } + if step == 0 { + return 0, fmt.Errorf("step of range should be a positive number: %s", expr) + } + + return u.getBits(start, end, step) | extra, nil +} + +// parseIntOrName returns the (possibly-named) integer contained in expr. +func (u *cronUtils) parseIntOrName(expr string, names map[string]uint) (uint, error) { + if names != nil { + if namedInt, ok := names[strings.ToLower(expr)]; ok { + return namedInt, nil + } + } + return u.mustParseInt(expr) +} + +// mustParseInt parses the given expression as an int or returns an error. +func (u *cronUtils) mustParseInt(expr string) (uint, error) { + num, err := strconv.Atoi(expr) + if err != nil { + return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err) + } + if num < 0 { + return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr) + } + + return uint(num), nil +} + +// getBits sets all bits in the range [min, max], modulo the given step size. +func (u *cronUtils) getBits(min, max, step uint) uint64 { + var bits uint64 + + // If step is 1, use shifts. + if step == 1 { + return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min) + } + + // Else, use a simple loop. + for i := min; i <= max; i += step { + bits |= 1 << i + } + return bits +} + +// all returns all bits within the given cronBounds. (plus the star bit) +func (u *cronUtils) all(r cronBounds) uint64 { + return u.getBits(r.min, r.max, 1) | CronUtils.starBit +} + +var CronUtils = cronUtils{ + // The cronBounds for each field. + seconds: cronBounds{0, 59, nil}, + minutes: cronBounds{0, 59, nil}, + hours: cronBounds{0, 23, nil}, + dom: cronBounds{1, 31, nil}, + months: cronBounds{1, 12, map[string]uint{ + "jan": 1, + "feb": 2, + "mar": 3, + "apr": 4, + "may": 5, + "jun": 6, + "jul": 7, + "aug": 8, + "sep": 9, + "oct": 10, + "nov": 11, + "dec": 12, + }}, + dow: cronBounds{0, 6, map[string]uint{ + "sun": 0, + "mon": 1, + "tue": 2, + "wed": 3, + "thu": 4, + "fri": 5, + "sat": 6, + }}, + + // Set the top bit if a star was included in the expression. + starBit: 1 << 63, +} diff --git a/core/utils/debug.go b/core/utils/debug.go new file mode 100644 index 00000000..09ef3098 --- /dev/null +++ b/core/utils/debug.go @@ -0,0 +1,18 @@ +package utils + +import ( + "fmt" + "github.com/spf13/viper" + "time" +) + +func IsDebug() bool { + return viper.GetBool("debug") +} + +func LogDebug(msg string) { + if !IsDebug() { + return + } + fmt.Println(fmt.Sprintf("[DEBUG] %s: %s", time.Now().Format("2006-01-02 15:04:05"), msg)) +} diff --git a/core/utils/demo.go b/core/utils/demo.go new file mode 100644 index 00000000..19e31dc5 --- /dev/null +++ b/core/utils/demo.go @@ -0,0 +1,57 @@ +package utils + +import ( + "fmt" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/sys_exec" + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" +) + +func GetApiAddress() (res string) { + apiAddress := viper.GetString("api.address") + if apiAddress == "" { + return "http://localhost:8000" + } + return apiAddress +} + +func IsDemo() (ok bool) { + return EnvIsTrue("demo", true) +} + +func InitializedDemo() (ok bool) { + col := mongo.GetMongoCol("users") + n, err := col.Count(nil) + if err != nil { + return true + } + return n > 0 +} + +func ImportDemo() (err error) { + cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo import", GetApiAddress()) + cmd := sys_exec.BuildCmd(cmdStr) + if err := cmd.Run(); err != nil { + trace.PrintError(err) + } + return nil +} + +func ReimportDemo() (err error) { + cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) + cmd := sys_exec.BuildCmd(cmdStr) + if err := cmd.Run(); err != nil { + trace.PrintError(err) + } + return nil +} + +func CleanupDemo() (err error) { + cmdStr := fmt.Sprintf("crawlab-cli login -a %s && crawlab-demo reimport", GetApiAddress()) + cmd := sys_exec.BuildCmd(cmdStr) + if err := cmd.Run(); err != nil { + trace.PrintError(err) + } + return nil +} diff --git a/core/utils/di.go b/core/utils/di.go new file mode 100644 index 00000000..8c467228 --- /dev/null +++ b/core/utils/di.go @@ -0,0 +1,18 @@ +package utils + +import ( + "github.com/crawlab-team/go-trace" + "github.com/spf13/viper" + "go.uber.org/dig" + "os" +) + +func VisualizeContainer(c *dig.Container) (err error) { + if !viper.GetBool("debug.di.visualize") { + return nil + } + if err := dig.Visualize(c, os.Stdout); err != nil { + return trace.TraceError(err) + } + return nil +} diff --git a/core/utils/docker.go b/core/utils/docker.go new file mode 100644 index 00000000..a1c98ed1 --- /dev/null +++ b/core/utils/docker.go @@ -0,0 +1,5 @@ +package utils + +func IsDocker() (ok bool) { + return EnvIsTrue("docker", false) +} diff --git a/core/utils/encrypt.go b/core/utils/encrypt.go new file mode 100644 index 00000000..d0754054 --- /dev/null +++ b/core/utils/encrypt.go @@ -0,0 +1,80 @@ +package utils + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/md5" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "io" +) + +func GetSecretKey() string { + return constants.DefaultEncryptServerKey +} + +func GetSecretKeyBytes() []byte { + return []byte(GetSecretKey()) +} + +func ComputeHmacSha256(message string, secret string) string { + key := []byte(secret) + h := hmac.New(sha256.New, key) + h.Write([]byte(message)) + sha := hex.EncodeToString(h.Sum(nil)) + return base64.StdEncoding.EncodeToString([]byte(sha)) +} + +func EncryptMd5(str string) string { + w := md5.New() + _, _ = io.WriteString(w, str) + md5str := fmt.Sprintf("%x", w.Sum(nil)) + return md5str +} + +func padding(src []byte, blockSize int) []byte { + padNum := blockSize - len(src)%blockSize + pad := bytes.Repeat([]byte{byte(padNum)}, padNum) + return append(src, pad...) +} + +func unPadding(src []byte) []byte { + n := len(src) + unPadNum := int(src[n-1]) + return src[:n-unPadNum] +} + +func EncryptAES(src string) (res string, err error) { + srcBytes := []byte(src) + key := GetSecretKeyBytes() + block, err := aes.NewCipher(key) + if err != nil { + return res, err + } + srcBytes = padding(srcBytes, block.BlockSize()) + blockMode := cipher.NewCBCEncrypter(block, key) + blockMode.CryptBlocks(srcBytes, srcBytes) + res = hex.EncodeToString(srcBytes) + return res, nil +} + +func DecryptAES(src string) (res string, err error) { + srcBytes, err := hex.DecodeString(src) + if err != nil { + return res, err + } + key := GetSecretKeyBytes() + block, err := aes.NewCipher(key) + if err != nil { + return res, err + } + blockMode := cipher.NewCBCDecrypter(block, key) + blockMode.CryptBlocks(srcBytes, srcBytes) + res = string(unPadding(srcBytes)) + return res, nil +} diff --git a/core/utils/encrypt_test.go b/core/utils/encrypt_test.go new file mode 100644 index 00000000..d393fa5b --- /dev/null +++ b/core/utils/encrypt_test.go @@ -0,0 +1,20 @@ +package utils + +import ( + "fmt" + "github.com/stretchr/testify/require" + "testing" +) + +func TestEncryptAesPassword(t *testing.T) { + plainText := "crawlab" + encryptedText, err := EncryptAES(plainText) + require.Nil(t, err) + decryptedText, err := DecryptAES(encryptedText) + require.Nil(t, err) + fmt.Println(fmt.Sprintf("plainText: %s", plainText)) + fmt.Println(fmt.Sprintf("encryptedText: %s", encryptedText)) + fmt.Println(fmt.Sprintf("decryptedText: %s", decryptedText)) + require.Equal(t, decryptedText, plainText) + require.NotEqual(t, decryptedText, encryptedText) +} diff --git a/core/utils/es.go b/core/utils/es.go new file mode 100644 index 00000000..587e1715 --- /dev/null +++ b/core/utils/es.go @@ -0,0 +1,159 @@ +package utils + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/cenkalti/backoff/v4" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/crawlab-team/go-trace" + "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/esapi" + "go.mongodb.org/mongo-driver/bson/primitive" + "time" +) + +func GetElasticsearchClient(ds *models.DataSource) (c *elasticsearch.Client, err error) { + return getElasticsearchClient(context.Background(), ds) +} + +func GetElasticsearchClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *elasticsearch.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getElasticsearchClient(ctx, ds) +} + +func getElasticsearchClient(ctx context.Context, ds *models.DataSource) (c *elasticsearch.Client, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultElasticsearchPort + } + + // es hosts + addresses := []string{ + fmt.Sprintf("http://%s:%s", host, port), + } + + // retry backoff + rb := backoff.NewExponentialBackOff() + + // es client options + cfg := elasticsearch.Config{ + Addresses: addresses, + Username: ds.Username, + Password: ds.Password, + //CloudID: "", + //APIKey: "", + //ServiceToken: "", + //CertificateFingerprint: "", + //Header: nil, + //CACert: nil, + //RetryOnStatus: nil, + //DisableRetry: false, + //EnableRetryOnTimeout: false, + //MaxRetries: 0, + //CompressRequestBody: false, + //DiscoverNodesOnStart: false, + //DiscoverNodesInterval: 0, + //EnableMetrics: false, + //EnableDebugLogger: false, + //EnableCompatibilityMode: false, + //DisableMetaHeader: false, + //UseResponseCheckOnly: false, + RetryBackoff: func(i int) time.Duration { + if i == 1 { + rb.Reset() + } + return rb.NextBackOff() + }, + //Transport: nil, + //Logger: nil, + //Selector: nil, + //ConnectionPoolFunc: nil, + } + + // es client + done := make(chan struct{}) + go func() { + c, err = elasticsearch.NewClient(cfg) + if err != nil { + return + } + var res *esapi.Response + res, err = c.Info() + fmt.Println(res) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return c, err +} + +func GetElasticsearchQuery(query generic.ListQuery) (buf *bytes.Buffer) { + q := map[string]interface{}{} + if len(query) > 0 { + match := getElasticsearchQueryMatch(query) + q["query"] = map[string]interface{}{ + "match": match, + } + } + buf = &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(q); err != nil { + trace.PrintError(err) + } + return buf +} + +func GetElasticsearchQueryWithOptions(query generic.ListQuery, opts *generic.ListOptions) (buf *bytes.Buffer) { + q := map[string]interface{}{ + "size": opts.Limit, + "from": opts.Skip, + // TODO: sort + } + if len(query) > 0 { + match := getElasticsearchQueryMatch(query) + q["query"] = map[string]interface{}{ + "match": match, + } + } + buf = &bytes.Buffer{} + if err := json.NewEncoder(buf).Encode(q); err != nil { + trace.PrintError(err) + } + return buf +} + +func getElasticsearchQueryMatch(query generic.ListQuery) (match map[string]interface{}) { + match = map[string]interface{}{} + for _, c := range query { + switch c.Value.(type) { + case primitive.ObjectID: + c.Value = c.Value.(primitive.ObjectID).Hex() + } + switch c.Op { + case generic.OpEqual: + match[c.Key] = c.Value + default: + match[c.Key] = map[string]interface{}{ + c.Op: c.Value, + } + } + } + return match +} diff --git a/core/utils/file.go b/core/utils/file.go new file mode 100644 index 00000000..216a129a --- /dev/null +++ b/core/utils/file.go @@ -0,0 +1,382 @@ +package utils + +import ( + "archive/zip" + "crypto/md5" + "encoding/hex" + "fmt" + "github.com/apex/log" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "runtime/debug" +) + +func OpenFile(fileName string) *os.File { + file, err := os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, os.ModePerm) + if err != nil { + log.Errorf("create file error: %s, file_name: %s", err.Error(), fileName) + debug.PrintStack() + return nil + } + return file +} + +func Exists(path string) bool { + _, err := os.Stat(path) //os.Stat获取文件信息 + if err != nil { + return os.IsExist(err) + } + return true +} + +func IsDir(path string) bool { + s, err := os.Stat(path) + if err != nil { + return false + } + return s.IsDir() +} + +// ListDir Add: 增加error类型作为第二返回值 +// 在其他函数如 /task/log/file_driver.go中的 *FileLogDriver.cleanup()函数调用时 +// 可以通过判断err是否为nil来判断是否有错误发生 +func ListDir(path string) ([]fs.FileInfo, error) { + list, err := os.ReadDir(path) + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return nil, err + } + + var res []fs.FileInfo + for _, item := range list { + info, err := item.Info() + if err != nil { + log.Errorf(err.Error()) + debug.PrintStack() + return nil, err + } + res = append(res, info) + } + return res, nil +} + +func DeCompress(srcFile *os.File, dstPath string) error { + // 如果保存路径不存在，创建一个 + if !Exists(dstPath) { + if err := os.MkdirAll(dstPath, os.ModePerm); err != nil { + debug.PrintStack() + return err + } + } + + // 读取zip文件 + zipFile, err := zip.OpenReader(srcFile.Name()) + if err != nil { + log.Errorf("Unzip File Error：" + err.Error()) + debug.PrintStack() + return err + } + defer Close(zipFile) + + // 遍历zip内所有文件和目录 + for _, innerFile := range zipFile.File { + // 获取该文件数据 + info := innerFile.FileInfo() + + // 如果是目录，则创建一个 + if info.IsDir() { + err = os.MkdirAll(filepath.Join(dstPath, innerFile.Name), os.ModeDir|os.ModePerm) + if err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + return err + } + continue + } + + // 如果文件目录不存在，则创建一个 + dirPath := filepath.Join(dstPath, filepath.Dir(innerFile.Name)) + if !Exists(dirPath) { + if err = os.MkdirAll(dirPath, os.ModeDir|os.ModePerm); err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + return err + } + } + + // 打开该文件 + srcFile, err := innerFile.Open() + if err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + continue + } + + // 创建新文件 + newFilePath := filepath.Join(dstPath, innerFile.Name) + newFile, err := os.OpenFile(newFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, info.Mode()) + if err != nil { + log.Errorf("Unzip File Error : " + err.Error()) + debug.PrintStack() + continue + } + + // 拷贝该文件到新文件中 + if _, err := io.Copy(newFile, srcFile); err != nil { + debug.PrintStack() + return err + } + + // 关闭该文件 + if err := srcFile.Close(); err != nil { + debug.PrintStack() + return err + } + + // 关闭新文件 + if err := newFile.Close(); err != nil { + debug.PrintStack() + return err + } + } + return nil +} + +// Compress 压缩文件 +// files 文件数组，可以是不同dir下的文件或者文件夹 +// dest 压缩文件存放地址 +func Compress(files []*os.File, dest string) error { + d, _ := os.Create(dest) + defer Close(d) + w := zip.NewWriter(d) + defer Close(w) + for _, file := range files { + if err := _Compress(file, "", w); err != nil { + return err + } + } + return nil +} + +func _Compress(file *os.File, prefix string, zw *zip.Writer) error { + info, err := file.Stat() + if err != nil { + debug.PrintStack() + return err + } + if info.IsDir() { + prefix = prefix + "/" + info.Name() + fileInfos, err := file.Readdir(-1) + if err != nil { + debug.PrintStack() + return err + } + for _, fi := range fileInfos { + f, err := os.Open(file.Name() + "/" + fi.Name()) + if err != nil { + debug.PrintStack() + return err + } + err = _Compress(f, prefix, zw) + if err != nil { + debug.PrintStack() + return err + } + } + } else { + header, err := zip.FileInfoHeader(info) + if err != nil { + debug.PrintStack() + return err + } + header.Name = prefix + "/" + header.Name + writer, err := zw.CreateHeader(header) + if err != nil { + debug.PrintStack() + return err + } + _, err = io.Copy(writer, file) + Close(file) + if err != nil { + debug.PrintStack() + return err + } + } + return nil +} + +func TrimFileData(data []byte) (res []byte) { + if string(data) == constants.EmptyFileData { + return res + } + return data +} + +func ZipDirectory(dir, zipfile string) error { + zipFile, err := os.Create(zipfile) + if err != nil { + return err + } + defer zipFile.Close() + + zipWriter := zip.NewWriter(zipFile) + defer zipWriter.Close() + + baseDir := filepath.Dir(dir) + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() { + return nil + } + + relPath, err := filepath.Rel(baseDir, path) + if err != nil { + return err + } + + zipFile, err := zipWriter.Create(relPath) + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(zipFile, file) + if err != nil { + return err + } + + return nil + }) + + return err +} + +// CopyFile File copies a single file from src to dst +func CopyFile(src, dst string) error { + var err error + var srcFd *os.File + var dstFd *os.File + var srcInfo os.FileInfo + + if srcFd, err = os.Open(src); err != nil { + return err + } + defer srcFd.Close() + + if dstFd, err = os.Create(dst); err != nil { + return err + } + defer dstFd.Close() + + if _, err = io.Copy(dstFd, srcFd); err != nil { + return err + } + if srcInfo, err = os.Stat(src); err != nil { + return err + } + return os.Chmod(dst, srcInfo.Mode()) +} + +// CopyDir Dir copies a whole directory recursively +func CopyDir(src string, dst string) error { + var err error + var fds []os.DirEntry + var srcInfo os.FileInfo + + if srcInfo, err = os.Stat(src); err != nil { + return err + } + + if err = os.MkdirAll(dst, srcInfo.Mode()); err != nil { + return err + } + + if fds, err = os.ReadDir(src); err != nil { + return err + } + for _, fd := range fds { + srcfp := path.Join(src, fd.Name()) + dstfp := path.Join(dst, fd.Name()) + + if fd.IsDir() { + if err = CopyDir(srcfp, dstfp); err != nil { + fmt.Println(err) + } + } else { + if err = CopyFile(srcfp, dstfp); err != nil { + fmt.Println(err) + } + } + } + return nil +} + +func GetFileHash(filePath string) (res string, err error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + defer file.Close() + + hash := md5.New() + if _, err := io.Copy(hash, file); err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func ScanDirectory(dir string) (res map[string]entity.FsFileInfo, err error) { + files := make(map[string]entity.FsFileInfo) + + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + hash, err := GetFileHash(path) + if err != nil { + return err + } + + relPath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + + files[relPath] = entity.FsFileInfo{ + Name: info.Name(), + Path: relPath, + FullPath: path, + Extension: filepath.Ext(path), + FileSize: info.Size(), + ModTime: info.ModTime(), + Mode: info.Mode(), + Hash: hash, + } + return nil + }) + if err != nil { + return nil, err + } + + return files, nil +} diff --git a/core/utils/file_test.go b/core/utils/file_test.go new file mode 100644 index 00000000..4af32d0d --- /dev/null +++ b/core/utils/file_test.go @@ -0,0 +1,129 @@ +package utils + +import ( + "archive/zip" + . "github.com/smartystreets/goconvey/convey" + "io" + "log" + "os" + "runtime/debug" + "testing" +) + +func TestExists(t *testing.T) { + var pathString = "../config" + var wrongPathString = "test" + + Convey("Test path or file is Exists or not", t, func() { + res := Exists(pathString) + Convey("The result should be true", func() { + So(res, ShouldEqual, true) + }) + wrongRes := Exists(wrongPathString) + Convey("The result should be false", func() { + So(wrongRes, ShouldEqual, false) + }) + }) +} + +func TestIsDir(t *testing.T) { + var pathString = "../config" + var fileString = "../config/config.go" + var wrongString = "test" + + Convey("Test path is folder or not", t, func() { + res := IsDir(pathString) + So(res, ShouldEqual, true) + fileRes := IsDir(fileString) + So(fileRes, ShouldEqual, false) + wrongRes := IsDir(wrongString) + So(wrongRes, ShouldEqual, false) + }) +} + +func TestCompress(t *testing.T) { + err := os.Mkdir("testCompress", os.ModePerm) + if err != nil { + t.Error("create testCompress failed") + } + var pathString = "testCompress" + var files []*os.File + var disPath = "testCompress" + file, err := os.Open(pathString) + if err != nil { + t.Error("open source path failed") + } + files = append(files, file) + Convey("Verify dispath is valid path", t, func() { + er := Compress(files, disPath) + Convey("err should be nil", func() { + So(er, ShouldEqual, nil) + }) + }) + _ = os.RemoveAll("testCompress") + +} +func Zip(zipFile string, fileList []string) error { + // 创建 zip 包文件 + fw, err := os.Create(zipFile) + if err != nil { + log.Fatal() + } + defer Close(fw) + + // 实例化新的 zip.Writer + zw := zip.NewWriter(fw) + defer Close(zw) + + for _, fileName := range fileList { + fr, err := os.Open(fileName) + if err != nil { + return err + } + fi, err := fr.Stat() + if err != nil { + return err + } + // 写入文件的头信息 + fh, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + w, err := zw.CreateHeader(fh) + if err != nil { + return err + } + // 写入文件内容 + _, err = io.Copy(w, fr) + if err != nil { + return err + } + } + return nil +} + +func TestDeCompress(t *testing.T) { + err := os.Mkdir("testDeCompress", os.ModePerm) + if err != nil { + t.Error(err) + + } + err = Zip("demo.zip", []string{}) + if err != nil { + t.Error("create zip file failed") + } + tmpFile, err := os.OpenFile("demo.zip", os.O_RDONLY, 0777) + if err != nil { + debug.PrintStack() + t.Error("open demo.zip failed") + } + var dstPath = "./testDeCompress" + Convey("Test DeCopmress func", t, func() { + + err := DeCompress(tmpFile, dstPath) + So(err, ShouldEqual, nil) + }) + _ = os.RemoveAll("testDeCompress") + _ = os.Remove("demo.zip") + +} diff --git a/core/utils/filter.go b/core/utils/filter.go new file mode 100644 index 00000000..13911bd3 --- /dev/null +++ b/core/utils/filter.go @@ -0,0 +1,49 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/errors" + "github.com/crawlab-team/crawlab/core/interfaces" + "go.mongodb.org/mongo-driver/bson" +) + +// FilterToQuery Translate entity.Filter to bson.M +func FilterToQuery(f interfaces.Filter) (q bson.M, err error) { + if f == nil || f.IsNil() { + return nil, nil + } + + q = bson.M{} + for _, cond := range f.GetConditions() { + key := cond.GetKey() + op := cond.GetOp() + value := cond.GetValue() + switch op { + case constants.FilterOpNotSet: + // do nothing + case constants.FilterOpEqual: + q[key] = cond.GetValue() + case constants.FilterOpNotEqual: + q[key] = bson.M{"$ne": value} + case constants.FilterOpContains, constants.FilterOpRegex, constants.FilterOpSearch: + q[key] = bson.M{"$regex": value, "$options": "i"} + case constants.FilterOpNotContains: + q[key] = bson.M{"$not": bson.M{"$regex": value}} + case constants.FilterOpIn: + q[key] = bson.M{"$in": value} + case constants.FilterOpNotIn: + q[key] = bson.M{"$nin": value} + case constants.FilterOpGreaterThan: + q[key] = bson.M{"$gt": value} + case constants.FilterOpGreaterThanEqual: + q[key] = bson.M{"$gte": value} + case constants.FilterOpLessThan: + q[key] = bson.M{"$lt": value} + case constants.FilterOpLessThanEqual: + q[key] = bson.M{"$lte": value} + default: + return nil, errors.ErrorFilterInvalidOperation + } + } + return q, nil +} diff --git a/core/utils/git.go b/core/utils/git.go new file mode 100644 index 00000000..1065aad5 --- /dev/null +++ b/core/utils/git.go @@ -0,0 +1,36 @@ +package utils + +import ( + vcs "github.com/crawlab-team/crawlab-vcs" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/interfaces" + "github.com/crawlab-team/crawlab/core/models/models" +) + +func InitGitClientAuth(g interfaces.Git, gitClient *vcs.GitClient) { + // set auth + switch g.GetAuthType() { + case constants.GitAuthTypeHttp: + gitClient.SetAuthType(vcs.GitAuthTypeHTTP) + gitClient.SetUsername(g.GetUsername()) + gitClient.SetPassword(g.GetPassword()) + case constants.GitAuthTypeSsh: + gitClient.SetAuthType(vcs.GitAuthTypeSSH) + gitClient.SetUsername(g.GetUsername()) + gitClient.SetPrivateKey(g.GetPassword()) + } +} + +func InitGitClientAuthV2(g *models.GitV2, gitClient *vcs.GitClient) { + // set auth + switch g.AuthType { + case constants.GitAuthTypeHttp: + gitClient.SetAuthType(vcs.GitAuthTypeHTTP) + gitClient.SetUsername(g.Username) + gitClient.SetPassword(g.Password) + case constants.GitAuthTypeSsh: + gitClient.SetAuthType(vcs.GitAuthTypeSSH) + gitClient.SetUsername(g.Username) + gitClient.SetPrivateKey(g.Password) + } +} diff --git a/core/utils/helpers.go b/core/utils/helpers.go new file mode 100644 index 00000000..338166f8 --- /dev/null +++ b/core/utils/helpers.go @@ -0,0 +1,36 @@ +package utils + +import ( + "github.com/crawlab-team/go-trace" + "io" + "reflect" + "unsafe" +) + +func BytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +func Close(c io.Closer) { + err := c.Close() + if err != nil { + trace.PrintError(err) + } +} + +func Contains(array interface{}, val interface{}) (fla bool) { + fla = false + switch reflect.TypeOf(array).Kind() { + case reflect.Slice: + { + s := reflect.ValueOf(array) + for i := 0; i < s.Len(); i++ { + if reflect.DeepEqual(val, s.Index(i).Interface()) { + fla = true + return + } + } + } + } + return +} diff --git a/core/utils/http.go b/core/utils/http.go new file mode 100644 index 00000000..3b2c0f66 --- /dev/null +++ b/core/utils/http.go @@ -0,0 +1,32 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/entity" + "github.com/crawlab-team/go-trace" + "github.com/gin-gonic/gin" + "net/http" +) + +func handleError(statusCode int, c *gin.Context, err error, print bool) { + if print { + trace.PrintError(err) + } + c.AbortWithStatusJSON(statusCode, entity.Response{ + Status: constants.HttpResponseStatusOk, + Message: constants.HttpResponseMessageError, + Error: err.Error(), + }) +} + +func HandleError(statusCode int, c *gin.Context, err error) { + handleError(statusCode, c, err, true) +} + +func HandleErrorUnauthorized(c *gin.Context, err error) { + HandleError(http.StatusUnauthorized, c, err) +} + +func HandleErrorInternalServerError(c *gin.Context, err error) { + HandleError(http.StatusInternalServerError, c, err) +} diff --git a/core/utils/init.go b/core/utils/init.go new file mode 100644 index 00000000..21a12bd8 --- /dev/null +++ b/core/utils/init.go @@ -0,0 +1,30 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab/core/interfaces" + "sync" +) + +var moduleInitializedMap = sync.Map{} + +func InitModule(id interfaces.ModuleId, fn func() error) (err error) { + res, ok := moduleInitializedMap.Load(id) + if ok { + initialized, _ := res.(bool) + if initialized { + return nil + } + } + + if err := fn(); err != nil { + return err + } + + moduleInitializedMap.Store(id, true) + + return nil +} + +func ForceInitModule(fn func() error) (err error) { + return fn() +} diff --git a/core/utils/json.go b/core/utils/json.go new file mode 100644 index 00000000..564a67eb --- /dev/null +++ b/core/utils/json.go @@ -0,0 +1,12 @@ +package utils + +import "encoding/json" + +func JsonToBytes(d interface{}) (bytes []byte, err error) { + switch d.(type) { + case []byte: + return d.([]byte), nil + default: + return json.Marshal(d) + } +} diff --git a/core/utils/kafka.go b/core/utils/kafka.go new file mode 100644 index 00000000..1bf005f2 --- /dev/null +++ b/core/utils/kafka.go @@ -0,0 +1,41 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/segmentio/kafka-go" + "time" +) + +func GetKafkaConnection(ds *models.DataSource) (c *kafka.Conn, err error) { + return getKafkaConnection(context.Background(), ds) +} + +func GetKafkaConnectionWithTimeout(ds *models.DataSource, timeout time.Duration) (c *kafka.Conn, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getKafkaConnection(ctx, ds) +} + +func getKafkaConnection(ctx context.Context, ds *models.DataSource) (c *kafka.Conn, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultKafkaPort + } + + // kafka connection address + network := "tcp" + address := fmt.Sprintf("%s:%s", host, port) + topic := ds.Database + partition := 0 // TODO: parameterize + + // kafka connection + return kafka.DialLeader(ctx, network, address, topic, partition) +} diff --git a/core/utils/mongo.go b/core/utils/mongo.go new file mode 100644 index 00000000..4b1ff502 --- /dev/null +++ b/core/utils/mongo.go @@ -0,0 +1,94 @@ +package utils + +import ( + "context" + "github.com/crawlab-team/crawlab-db/generic" + "github.com/crawlab-team/crawlab-db/mongo" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "go.mongodb.org/mongo-driver/bson" + mongo2 "go.mongodb.org/mongo-driver/mongo" + "time" +) + +func GetMongoQuery(query generic.ListQuery) (res bson.M) { + res = bson.M{} + for _, c := range query { + switch c.Op { + case generic.OpEqual: + res[c.Key] = c.Value + default: + res[c.Key] = bson.M{ + c.Op: c.Value, + } + } + } + return res +} + +func GetMongoOpts(opts *generic.ListOptions) (res *mongo.FindOptions) { + var sort bson.D + for _, s := range opts.Sort { + direction := 1 + if s.Direction == generic.SortDirectionAsc { + direction = 1 + } else if s.Direction == generic.SortDirectionDesc { + direction = -1 + } + sort = append(sort, bson.E{Key: s.Key, Value: direction}) + } + return &mongo.FindOptions{ + Skip: opts.Skip, + Limit: opts.Limit, + Sort: sort, + } +} + +func GetMongoClient(ds *models.DataSource) (c *mongo2.Client, err error) { + return getMongoClient(context.Background(), ds) +} + +func GetMongoClientWithTimeout(ds *models.DataSource, timeout time.Duration) (c *mongo2.Client, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMongoClient(ctx, ds) +} + +func getMongoClient(ctx context.Context, ds *models.DataSource) (c *mongo2.Client, err error) { + // normalize settings + if ds.Host == "" { + ds.Host = constants.DefaultHost + } + if ds.Port == "" { + ds.Port = constants.DefaultMongoPort + } + + // options + var opts []mongo.ClientOption + opts = append(opts, mongo.WithContext(ctx)) + opts = append(opts, mongo.WithUri(ds.Url)) + opts = append(opts, mongo.WithHost(ds.Host)) + opts = append(opts, mongo.WithPort(ds.Port)) + opts = append(opts, mongo.WithDb(ds.Database)) + opts = append(opts, mongo.WithUsername(ds.Username)) + opts = append(opts, mongo.WithPassword(ds.Password)) + opts = append(opts, mongo.WithHosts(ds.Hosts)) + + // extra + if ds.Extra != nil { + // auth source + authSource, ok := ds.Extra["auth_source"] + if ok { + opts = append(opts, mongo.WithAuthSource(authSource)) + } + + // auth mechanism + authMechanism, ok := ds.Extra["auth_mechanism"] + if ok { + opts = append(opts, mongo.WithAuthMechanism(authMechanism)) + } + } + + // client + return mongo.GetMongoClient(opts...) +} diff --git a/core/utils/mssql.go b/core/utils/mssql.go new file mode 100644 index 00000000..0fb21353 --- /dev/null +++ b/core/utils/mssql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mssql" + "time" +) + +func GetMssqlSession(ds *models.DataSource) (s db.Session, err error) { + return getMssqlSession(context.Background(), ds) +} + +func GetMssqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMssqlSession(ctx, ds) +} + +func getMssqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMssqlPort + } + + // connect settings + settings := mssql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mssql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/mysql.go b/core/utils/mysql.go new file mode 100644 index 00000000..8fc9ae18 --- /dev/null +++ b/core/utils/mysql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/mysql" + "time" +) + +func GetMysqlSession(ds *models.DataSource) (s db.Session, err error) { + return getMysqlSession(context.Background(), ds) +} + +func GetMysqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getMysqlSession(ctx, ds) +} + +func getMysqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultMysqlPort + } + + // connect settings + settings := mysql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = mysql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/node.go b/core/utils/node.go new file mode 100644 index 00000000..1f20ade1 --- /dev/null +++ b/core/utils/node.go @@ -0,0 +1,13 @@ +package utils + +func IsMaster() bool { + return EnvIsTrue("node.master", false) +} + +func GetNodeType() string { + if IsMaster() { + return "master" + } else { + return "worker" + } +} diff --git a/core/utils/os.go b/core/utils/os.go new file mode 100644 index 00000000..18351252 --- /dev/null +++ b/core/utils/os.go @@ -0,0 +1,13 @@ +package utils + +import ( + "os" + "os/signal" + "syscall" +) + +func DefaultWait() { + quit := make(chan os.Signal, 1) + signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + <-quit +} diff --git a/core/utils/postgresql.go b/core/utils/postgresql.go new file mode 100644 index 00000000..40c8a208 --- /dev/null +++ b/core/utils/postgresql.go @@ -0,0 +1,60 @@ +package utils + +import ( + "context" + "fmt" + "github.com/crawlab-team/crawlab/core/constants" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/postgresql" + "time" +) + +func GetPostgresqlSession(ds *models.DataSource) (s db.Session, err error) { + return getPostgresqlSession(context.Background(), ds) +} + +func GetPostgresqlSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getPostgresqlSession(ctx, ds) +} + +func getPostgresqlSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // normalize settings + host := ds.Host + port := ds.Port + if ds.Host == "" { + host = constants.DefaultHost + } + if ds.Port == "" { + port = constants.DefaultPostgresqlPort + } + + // connect settings + settings := postgresql.ConnectionURL{ + User: ds.Username, + Password: ds.Password, + Database: ds.Database, + Host: fmt.Sprintf("%s:%s", host, port), + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = postgresql.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/result.go b/core/utils/result.go new file mode 100644 index 00000000..62450309 --- /dev/null +++ b/core/utils/result.go @@ -0,0 +1,23 @@ +package utils + +import ( + "encoding/json" + "github.com/crawlab-team/crawlab/core/interfaces" +) + +func GetResultHash(value interface{}, keys []string) (res string, err error) { + m := make(map[string]interface{}) + for _, k := range keys { + _value, ok := value.(interfaces.Result) + if !ok { + continue + } + v := _value.GetValue(k) + m[k] = v + } + data, err := json.Marshal(m) + if err != nil { + return "", err + } + return EncryptMd5(string(data)), nil +} diff --git a/core/utils/rpc.go b/core/utils/rpc.go new file mode 100644 index 00000000..03414199 --- /dev/null +++ b/core/utils/rpc.go @@ -0,0 +1,14 @@ +package utils + +import "encoding/json" + +// Object 转化为 String +func ObjectToString(params interface{}) string { + bytes, _ := json.Marshal(params) + return BytesToString(bytes) +} + +// 获取 RPC 参数 +func GetRpcParam(key string, params map[string]string) string { + return params[key] +} diff --git a/core/utils/spider.go b/core/utils/spider.go new file mode 100644 index 00000000..4484ccf0 --- /dev/null +++ b/core/utils/spider.go @@ -0,0 +1,8 @@ +package utils + +func GetSpiderCol(col string, name string) string { + if col == "" { + return "results_" + name + } + return col +} diff --git a/core/utils/sql.go b/core/utils/sql.go new file mode 100644 index 00000000..f231d755 --- /dev/null +++ b/core/utils/sql.go @@ -0,0 +1,27 @@ +package utils + +import ( + "github.com/crawlab-team/crawlab-db/generic" + "github.com/upper/db/v4" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +func GetSqlQuery(query generic.ListQuery) (res db.Cond) { + res = db.Cond{} + for _, c := range query { + switch c.Value.(type) { + case primitive.ObjectID: + c.Value = c.Value.(primitive.ObjectID).Hex() + } + switch c.Op { + case generic.OpEqual: + res[c.Key] = c.Value + default: + res[c.Key] = db.Cond{ + c.Op: c.Value, + } + } + } + // TODO: sort + return res +} diff --git a/core/utils/sqlite.go b/core/utils/sqlite.go new file mode 100644 index 00000000..1d6ff682 --- /dev/null +++ b/core/utils/sqlite.go @@ -0,0 +1,45 @@ +package utils + +import ( + "context" + "github.com/crawlab-team/crawlab/core/models/models" + "github.com/upper/db/v4" + "github.com/upper/db/v4/adapter/sqlite" + "time" +) + +func GetSqliteSession(ds *models.DataSource) (s db.Session, err error) { + return getSqliteSession(context.Background(), ds) +} + +func GetSqliteSessionWithTimeout(ds *models.DataSource, timeout time.Duration) (s db.Session, err error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + return getSqliteSession(ctx, ds) +} + +func getSqliteSession(ctx context.Context, ds *models.DataSource) (s db.Session, err error) { + // connect settings + settings := sqlite.ConnectionURL{ + Database: ds.Database, + Options: nil, + } + + // session + done := make(chan struct{}) + go func() { + s, err = sqlite.Open(settings) + close(done) + }() + + // wait for done + select { + case <-ctx.Done(): + if ctx.Err() != nil { + err = ctx.Err() + } + case <-done: + } + + return s, err +} diff --git a/core/utils/stats.go b/core/utils/stats.go new file mode 100644 index 00000000..d4b585bf --- /dev/null +++ b/core/utils/stats.go @@ -0,0 +1 @@ +package utils diff --git a/core/utils/system.go b/core/utils/system.go new file mode 100644 index 00000000..9b161f11 --- /dev/null +++ b/core/utils/system.go @@ -0,0 +1,7 @@ +package utils + +import "github.com/spf13/viper" + +func IsPro() bool { + return viper.GetString("info.edition") == "global.edition.pro" +} diff --git a/core/utils/task.go b/core/utils/task.go new file mode 100644 index 00000000..94bfbb3b --- /dev/null +++ b/core/utils/task.go @@ -0,0 +1,13 @@ +package utils + +import "github.com/crawlab-team/crawlab/core/constants" + +func IsCancellable(status string) bool { + switch status { + case constants.TaskStatusPending, + constants.TaskStatusRunning: + return true + default: + return false + } +} diff --git a/core/utils/time.go b/core/utils/time.go new file mode 100644 index 00000000..689e2c28 --- /dev/null +++ b/core/utils/time.go @@ -0,0 +1,18 @@ +package utils + +import ( + "time" +) + +func GetLocalTime(t time.Time) time.Time { + return t.In(time.Local) +} + +func GetTimeString(t time.Time) string { + return t.Format("2006-01-02 15:04:05") +} + +func GetLocalTimeString(t time.Time) string { + t = GetLocalTime(t) + return GetTimeString(t) +} diff --git a/core/utils/uuid.go b/core/utils/uuid.go new file mode 100644 index 00000000..72c5191c --- /dev/null +++ b/core/utils/uuid.go @@ -0,0 +1,8 @@ +package utils + +import "github.com/google/uuid" + +func NewUUIDString() (res string) { + id, _ := uuid.NewUUID() + return id.String() +} diff --git a/grpc/.gitignore b/grpc/.gitignore new file mode 100644 index 00000000..f32e31af --- /dev/null +++ b/grpc/.gitignore @@ -0,0 +1,2 @@ +.idea/ +.DS_Store diff --git a/grpc/LICENSE b/grpc/LICENSE new file mode 100644 index 00000000..3c0946b9 --- /dev/null +++ b/grpc/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2021, Crawlab Team +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/grpc/README.md b/grpc/README.md new file mode 100644 index 00000000..207b4216 --- /dev/null +++ b/grpc/README.md @@ -0,0 +1,2 @@ +# crawlab-grpc +gRPC for Crawlab diff --git a/grpc/bin/compile.sh b/grpc/bin/compile.sh new file mode 100755 index 00000000..b0d5c65f --- /dev/null +++ b/grpc/bin/compile.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +if [ -L $0 ] +then + BASE_DIR=`dirname $(readlink $0)` +else + BASE_DIR=`dirname $0` +fi +base_path=$(cd $BASE_DIR/..; pwd) + +cd $base_path && \ + rm -rf dist | true + +cd $base_path && \ + mkdir -p dist/python | true && \ + mkdir -p dist/js | true && \ + mkdir -p dist/ts | true && \ + mkdir -p dist/java | true && \ + mkdir -p dist/csharp | true && \ + mkdir -p dist/php | true && \ + mkdir -p dist/ruby | true + +cd $base_path && \ + protoc -I ./proto \ + --go_out=. \ + --go-grpc_out=. \ + --python_out=dist/python \ + --js_out=dist/js \ + --java_out=dist/java \ + --csharp_out=dist/csharp \ + ./proto/**/*.proto + +# python +cd $base_path && \ + python3 -m grpc_tools.protoc -I ./proto \ + --grpc_python_out=dist/python \ + ./proto/**/*.proto diff --git a/grpc/dependencies_service_v2_request.pb.go b/grpc/dependencies_service_v2_request.pb.go new file mode 100644 index 00000000..cae03422 --- /dev/null +++ b/grpc/dependencies_service_v2_request.pb.go @@ -0,0 +1,488 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/dependencies_service_v2_request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Dependency struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` +} + +func (x *Dependency) Reset() { + *x = Dependency{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Dependency) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Dependency) ProtoMessage() {} + +func (x *Dependency) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Dependency.ProtoReflect.Descriptor instead. +func (*Dependency) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{0} +} + +func (x *Dependency) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Dependency) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +type DependenciesServiceV2ConnectRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` +} + +func (x *DependenciesServiceV2ConnectRequest) Reset() { + *x = DependenciesServiceV2ConnectRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2ConnectRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2ConnectRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2ConnectRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2ConnectRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2ConnectRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{1} +} + +func (x *DependenciesServiceV2ConnectRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +type DependenciesServiceV2SyncRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` +} + +func (x *DependenciesServiceV2SyncRequest) Reset() { + *x = DependenciesServiceV2SyncRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2SyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2SyncRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2SyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2SyncRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2SyncRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{2} +} + +func (x *DependenciesServiceV2SyncRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2SyncRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2SyncRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +type DependenciesServiceV2InstallRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` + Proxy string `protobuf:"bytes,4,opt,name=proxy,proto3" json:"proxy,omitempty"` +} + +func (x *DependenciesServiceV2InstallRequest) Reset() { + *x = DependenciesServiceV2InstallRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2InstallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2InstallRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2InstallRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2InstallRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2InstallRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{3} +} + +func (x *DependenciesServiceV2InstallRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2InstallRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2InstallRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +func (x *DependenciesServiceV2InstallRequest) GetProxy() string { + if x != nil { + return x.Proxy + } + return "" +} + +type DependenciesServiceV2UninstallRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Lang string `protobuf:"bytes,2,opt,name=lang,proto3" json:"lang,omitempty"` + Dependencies []*Dependency `protobuf:"bytes,3,rep,name=dependencies,proto3" json:"dependencies,omitempty"` +} + +func (x *DependenciesServiceV2UninstallRequest) Reset() { + *x = DependenciesServiceV2UninstallRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DependenciesServiceV2UninstallRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DependenciesServiceV2UninstallRequest) ProtoMessage() {} + +func (x *DependenciesServiceV2UninstallRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_dependencies_service_v2_request_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DependenciesServiceV2UninstallRequest.ProtoReflect.Descriptor instead. +func (*DependenciesServiceV2UninstallRequest) Descriptor() ([]byte, []int) { + return file_entity_dependencies_service_v2_request_proto_rawDescGZIP(), []int{4} +} + +func (x *DependenciesServiceV2UninstallRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *DependenciesServiceV2UninstallRequest) GetLang() string { + if x != nil { + return x.Lang + } + return "" +} + +func (x *DependenciesServiceV2UninstallRequest) GetDependencies() []*Dependency { + if x != nil { + return x.Dependencies + } + return nil +} + +var File_entity_dependencies_service_v2_request_proto protoreflect.FileDescriptor + +var file_entity_dependencies_service_v2_request_proto_rawDesc = []byte{ + 0x0a, 0x2c, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, + 0x6e, 0x63, 0x69, 0x65, 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, + 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, + 0x67, 0x72, 0x70, 0x63, 0x22, 0x3a, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0x40, 0x0a, 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x22, 0x87, 0x01, 0x0a, 0x20, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, + 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, + 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x22, 0xa0, 0x01, 0x0a, + 0x23, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6c, + 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, + 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x0c, 0x64, 0x65, 0x70, + 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x72, 0x6f, + 0x78, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x22, + 0x8c, 0x01, 0x0a, 0x25, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, + 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, + 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, + 0x65, 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6c, 0x61, 0x6e, 0x67, 0x12, 0x34, 0x0a, 0x0c, 0x64, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, + 0x52, 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x42, 0x08, + 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_dependencies_service_v2_request_proto_rawDescOnce sync.Once + file_entity_dependencies_service_v2_request_proto_rawDescData = file_entity_dependencies_service_v2_request_proto_rawDesc +) + +func file_entity_dependencies_service_v2_request_proto_rawDescGZIP() []byte { + file_entity_dependencies_service_v2_request_proto_rawDescOnce.Do(func() { + file_entity_dependencies_service_v2_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_dependencies_service_v2_request_proto_rawDescData) + }) + return file_entity_dependencies_service_v2_request_proto_rawDescData +} + +var file_entity_dependencies_service_v2_request_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_entity_dependencies_service_v2_request_proto_goTypes = []interface{}{ + (*Dependency)(nil), // 0: grpc.Dependency + (*DependenciesServiceV2ConnectRequest)(nil), // 1: grpc.DependenciesServiceV2ConnectRequest + (*DependenciesServiceV2SyncRequest)(nil), // 2: grpc.DependenciesServiceV2SyncRequest + (*DependenciesServiceV2InstallRequest)(nil), // 3: grpc.DependenciesServiceV2InstallRequest + (*DependenciesServiceV2UninstallRequest)(nil), // 4: grpc.DependenciesServiceV2UninstallRequest +} +var file_entity_dependencies_service_v2_request_proto_depIdxs = []int32{ + 0, // 0: grpc.DependenciesServiceV2SyncRequest.dependencies:type_name -> grpc.Dependency + 0, // 1: grpc.DependenciesServiceV2InstallRequest.dependencies:type_name -> grpc.Dependency + 0, // 2: grpc.DependenciesServiceV2UninstallRequest.dependencies:type_name -> grpc.Dependency + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_entity_dependencies_service_v2_request_proto_init() } +func file_entity_dependencies_service_v2_request_proto_init() { + if File_entity_dependencies_service_v2_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_dependencies_service_v2_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Dependency); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2ConnectRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2SyncRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2InstallRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_dependencies_service_v2_request_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DependenciesServiceV2UninstallRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_dependencies_service_v2_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_dependencies_service_v2_request_proto_goTypes, + DependencyIndexes: file_entity_dependencies_service_v2_request_proto_depIdxs, + MessageInfos: file_entity_dependencies_service_v2_request_proto_msgTypes, + }.Build() + File_entity_dependencies_service_v2_request_proto = out.File + file_entity_dependencies_service_v2_request_proto_rawDesc = nil + file_entity_dependencies_service_v2_request_proto_goTypes = nil + file_entity_dependencies_service_v2_request_proto_depIdxs = nil +} diff --git a/grpc/dependency_service_v2.pb.go b/grpc/dependency_service_v2.pb.go new file mode 100644 index 00000000..7057a9cb --- /dev/null +++ b/grpc/dependency_service_v2.pb.go @@ -0,0 +1,108 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/dependency_service_v2.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_dependency_service_v2_proto protoreflect.FileDescriptor + +var file_services_dependency_service_v2_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, + 0x64, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x2c, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x2f, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, + 0x73, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, + 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x32, 0xc9, 0x02, 0x0a, 0x13, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x12, 0x48, 0x0a, 0x07, 0x43, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x28, 0x01, 0x12, 0x40, 0x0a, 0x04, 0x53, 0x79, 0x6e, 0x63, 0x12, 0x26, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x53, 0x79, 0x6e, 0x63, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4a, 0x0a, 0x07, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x12, 0x29, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, + 0x74, 0x61, 0x6c, 0x6c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, + 0x01, 0x12, 0x5a, 0x0a, 0x15, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, 0x44, 0x65, + 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x2b, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x6e, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6c, 0x6c, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, 0x5a, + 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_dependency_service_v2_proto_goTypes = []interface{}{ + (*DependenciesServiceV2ConnectRequest)(nil), // 0: grpc.DependenciesServiceV2ConnectRequest + (*DependenciesServiceV2SyncRequest)(nil), // 1: grpc.DependenciesServiceV2SyncRequest + (*DependenciesServiceV2InstallRequest)(nil), // 2: grpc.DependenciesServiceV2InstallRequest + (*DependenciesServiceV2UninstallRequest)(nil), // 3: grpc.DependenciesServiceV2UninstallRequest + (*Response)(nil), // 4: grpc.Response +} +var file_services_dependency_service_v2_proto_depIdxs = []int32{ + 0, // 0: grpc.DependencyServiceV2.Connect:input_type -> grpc.DependenciesServiceV2ConnectRequest + 1, // 1: grpc.DependencyServiceV2.Sync:input_type -> grpc.DependenciesServiceV2SyncRequest + 2, // 2: grpc.DependencyServiceV2.Install:input_type -> grpc.DependenciesServiceV2InstallRequest + 3, // 3: grpc.DependencyServiceV2.UninstallDependencies:input_type -> grpc.DependenciesServiceV2UninstallRequest + 4, // 4: grpc.DependencyServiceV2.Connect:output_type -> grpc.Response + 4, // 5: grpc.DependencyServiceV2.Sync:output_type -> grpc.Response + 4, // 6: grpc.DependencyServiceV2.Install:output_type -> grpc.Response + 4, // 7: grpc.DependencyServiceV2.UninstallDependencies:output_type -> grpc.Response + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_dependency_service_v2_proto_init() } +func file_services_dependency_service_v2_proto_init() { + if File_services_dependency_service_v2_proto != nil { + return + } + file_entity_dependencies_service_v2_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_dependency_service_v2_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_dependency_service_v2_proto_goTypes, + DependencyIndexes: file_services_dependency_service_v2_proto_depIdxs, + }.Build() + File_services_dependency_service_v2_proto = out.File + file_services_dependency_service_v2_proto_rawDesc = nil + file_services_dependency_service_v2_proto_goTypes = nil + file_services_dependency_service_v2_proto_depIdxs = nil +} diff --git a/grpc/dependency_service_v2_grpc.pb.go b/grpc/dependency_service_v2_grpc.pb.go new file mode 100644 index 00000000..3d0a9eb9 --- /dev/null +++ b/grpc/dependency_service_v2_grpc.pb.go @@ -0,0 +1,312 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/dependency_service_v2.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// DependencyServiceV2Client is the client API for DependencyServiceV2 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type DependencyServiceV2Client interface { + Connect(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_ConnectClient, error) + Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) + Install(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_InstallClient, error) + UninstallDependencies(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_UninstallDependenciesClient, error) +} + +type dependencyServiceV2Client struct { + cc grpc.ClientConnInterface +} + +func NewDependencyServiceV2Client(cc grpc.ClientConnInterface) DependencyServiceV2Client { + return &dependencyServiceV2Client{cc} +} + +func (c *dependencyServiceV2Client) Connect(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_ConnectClient, error) { + stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[0], "/grpc.DependencyServiceV2/Connect", opts...) + if err != nil { + return nil, err + } + x := &dependencyServiceV2ConnectClient{stream} + return x, nil +} + +type DependencyServiceV2_ConnectClient interface { + Send(*DependenciesServiceV2ConnectRequest) error + CloseAndRecv() (*Response, error) + grpc.ClientStream +} + +type dependencyServiceV2ConnectClient struct { + grpc.ClientStream +} + +func (x *dependencyServiceV2ConnectClient) Send(m *DependenciesServiceV2ConnectRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependencyServiceV2ConnectClient) CloseAndRecv() (*Response, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dependencyServiceV2Client) Sync(ctx context.Context, in *DependenciesServiceV2SyncRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.DependencyServiceV2/Sync", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *dependencyServiceV2Client) Install(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_InstallClient, error) { + stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[1], "/grpc.DependencyServiceV2/Install", opts...) + if err != nil { + return nil, err + } + x := &dependencyServiceV2InstallClient{stream} + return x, nil +} + +type DependencyServiceV2_InstallClient interface { + Send(*DependenciesServiceV2InstallRequest) error + Recv() (*Response, error) + grpc.ClientStream +} + +type dependencyServiceV2InstallClient struct { + grpc.ClientStream +} + +func (x *dependencyServiceV2InstallClient) Send(m *DependenciesServiceV2InstallRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependencyServiceV2InstallClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *dependencyServiceV2Client) UninstallDependencies(ctx context.Context, opts ...grpc.CallOption) (DependencyServiceV2_UninstallDependenciesClient, error) { + stream, err := c.cc.NewStream(ctx, &DependencyServiceV2_ServiceDesc.Streams[2], "/grpc.DependencyServiceV2/UninstallDependencies", opts...) + if err != nil { + return nil, err + } + x := &dependencyServiceV2UninstallDependenciesClient{stream} + return x, nil +} + +type DependencyServiceV2_UninstallDependenciesClient interface { + Send(*DependenciesServiceV2UninstallRequest) error + Recv() (*Response, error) + grpc.ClientStream +} + +type dependencyServiceV2UninstallDependenciesClient struct { + grpc.ClientStream +} + +func (x *dependencyServiceV2UninstallDependenciesClient) Send(m *DependenciesServiceV2UninstallRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *dependencyServiceV2UninstallDependenciesClient) Recv() (*Response, error) { + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DependencyServiceV2Server is the server API for DependencyServiceV2 service. +// All implementations must embed UnimplementedDependencyServiceV2Server +// for forward compatibility +type DependencyServiceV2Server interface { + Connect(DependencyServiceV2_ConnectServer) error + Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) + Install(DependencyServiceV2_InstallServer) error + UninstallDependencies(DependencyServiceV2_UninstallDependenciesServer) error + mustEmbedUnimplementedDependencyServiceV2Server() +} + +// UnimplementedDependencyServiceV2Server must be embedded to have forward compatible implementations. +type UnimplementedDependencyServiceV2Server struct { +} + +func (UnimplementedDependencyServiceV2Server) Connect(DependencyServiceV2_ConnectServer) error { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedDependencyServiceV2Server) Sync(context.Context, *DependenciesServiceV2SyncRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Sync not implemented") +} +func (UnimplementedDependencyServiceV2Server) Install(DependencyServiceV2_InstallServer) error { + return status.Errorf(codes.Unimplemented, "method Install not implemented") +} +func (UnimplementedDependencyServiceV2Server) UninstallDependencies(DependencyServiceV2_UninstallDependenciesServer) error { + return status.Errorf(codes.Unimplemented, "method UninstallDependencies not implemented") +} +func (UnimplementedDependencyServiceV2Server) mustEmbedUnimplementedDependencyServiceV2Server() {} + +// UnsafeDependencyServiceV2Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to DependencyServiceV2Server will +// result in compilation errors. +type UnsafeDependencyServiceV2Server interface { + mustEmbedUnimplementedDependencyServiceV2Server() +} + +func RegisterDependencyServiceV2Server(s grpc.ServiceRegistrar, srv DependencyServiceV2Server) { + s.RegisterService(&DependencyServiceV2_ServiceDesc, srv) +} + +func _DependencyServiceV2_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependencyServiceV2Server).Connect(&dependencyServiceV2ConnectServer{stream}) +} + +type DependencyServiceV2_ConnectServer interface { + SendAndClose(*Response) error + Recv() (*DependenciesServiceV2ConnectRequest, error) + grpc.ServerStream +} + +type dependencyServiceV2ConnectServer struct { + grpc.ServerStream +} + +func (x *dependencyServiceV2ConnectServer) SendAndClose(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependencyServiceV2ConnectServer) Recv() (*DependenciesServiceV2ConnectRequest, error) { + m := new(DependenciesServiceV2ConnectRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DependencyServiceV2_Sync_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DependenciesServiceV2SyncRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DependencyServiceV2Server).Sync(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.DependencyServiceV2/Sync", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DependencyServiceV2Server).Sync(ctx, req.(*DependenciesServiceV2SyncRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _DependencyServiceV2_Install_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependencyServiceV2Server).Install(&dependencyServiceV2InstallServer{stream}) +} + +type DependencyServiceV2_InstallServer interface { + Send(*Response) error + Recv() (*DependenciesServiceV2InstallRequest, error) + grpc.ServerStream +} + +type dependencyServiceV2InstallServer struct { + grpc.ServerStream +} + +func (x *dependencyServiceV2InstallServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependencyServiceV2InstallServer) Recv() (*DependenciesServiceV2InstallRequest, error) { + m := new(DependenciesServiceV2InstallRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _DependencyServiceV2_UninstallDependencies_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(DependencyServiceV2Server).UninstallDependencies(&dependencyServiceV2UninstallDependenciesServer{stream}) +} + +type DependencyServiceV2_UninstallDependenciesServer interface { + Send(*Response) error + Recv() (*DependenciesServiceV2UninstallRequest, error) + grpc.ServerStream +} + +type dependencyServiceV2UninstallDependenciesServer struct { + grpc.ServerStream +} + +func (x *dependencyServiceV2UninstallDependenciesServer) Send(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *dependencyServiceV2UninstallDependenciesServer) Recv() (*DependenciesServiceV2UninstallRequest, error) { + m := new(DependenciesServiceV2UninstallRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// DependencyServiceV2_ServiceDesc is the grpc.ServiceDesc for DependencyServiceV2 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var DependencyServiceV2_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.DependencyServiceV2", + HandlerType: (*DependencyServiceV2Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Sync", + Handler: _DependencyServiceV2_Sync_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _DependencyServiceV2_Connect_Handler, + ClientStreams: true, + }, + { + StreamName: "Install", + Handler: _DependencyServiceV2_Install_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "UninstallDependencies", + Handler: _DependencyServiceV2_UninstallDependencies_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/dependency_service_v2.proto", +} diff --git a/grpc/go.mod b/grpc/go.mod new file mode 100644 index 00000000..bbc0968c --- /dev/null +++ b/grpc/go.mod @@ -0,0 +1,16 @@ +module github.com/crawlab-team/crawlab/grpc + +go 1.22 + +require ( + github.com/golang/protobuf v1.5.4 + google.golang.org/grpc v1.64.0 + google.golang.org/protobuf v1.34.1 +) + +require ( + golang.org/x/net v0.22.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect +) diff --git a/grpc/go.sum b/grpc/go.sum new file mode 100644 index 00000000..d41d2dc2 --- /dev/null +++ b/grpc/go.sum @@ -0,0 +1,16 @@ +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= diff --git a/grpc/message_service.pb.go b/grpc/message_service.pb.go new file mode 100644 index 00000000..fca04128 --- /dev/null +++ b/grpc/message_service.pb.go @@ -0,0 +1,79 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/message_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_message_service_proto protoreflect.FileDescriptor + +var file_services_message_service_proto_rawDesc = []byte{ + 0x0a, 0x1e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x32, 0x4b, 0x0a, 0x0e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x39, 0x0a, 0x07, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, + 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, +} + +var file_services_message_service_proto_goTypes = []interface{}{ + (*StreamMessage)(nil), // 0: grpc.StreamMessage +} +var file_services_message_service_proto_depIdxs = []int32{ + 0, // 0: grpc.MessageService.Connect:input_type -> grpc.StreamMessage + 0, // 1: grpc.MessageService.Connect:output_type -> grpc.StreamMessage + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_message_service_proto_init() } +func file_services_message_service_proto_init() { + if File_services_message_service_proto != nil { + return + } + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_message_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_message_service_proto_goTypes, + DependencyIndexes: file_services_message_service_proto_depIdxs, + }.Build() + File_services_message_service_proto = out.File + file_services_message_service_proto_rawDesc = nil + file_services_message_service_proto_goTypes = nil + file_services_message_service_proto_depIdxs = nil +} diff --git a/grpc/message_service_grpc.pb.go b/grpc/message_service_grpc.pb.go new file mode 100644 index 00000000..c1cc17b0 --- /dev/null +++ b/grpc/message_service_grpc.pb.go @@ -0,0 +1,137 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/message_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// MessageServiceClient is the client API for MessageService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type MessageServiceClient interface { + Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) +} + +type messageServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewMessageServiceClient(cc grpc.ClientConnInterface) MessageServiceClient { + return &messageServiceClient{cc} +} + +func (c *messageServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (MessageService_ConnectClient, error) { + stream, err := c.cc.NewStream(ctx, &MessageService_ServiceDesc.Streams[0], "/grpc.MessageService/Connect", opts...) + if err != nil { + return nil, err + } + x := &messageServiceConnectClient{stream} + return x, nil +} + +type MessageService_ConnectClient interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type messageServiceConnectClient struct { + grpc.ClientStream +} + +func (x *messageServiceConnectClient) Send(m *StreamMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *messageServiceConnectClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MessageServiceServer is the server API for MessageService service. +// All implementations must embed UnimplementedMessageServiceServer +// for forward compatibility +type MessageServiceServer interface { + Connect(MessageService_ConnectServer) error + mustEmbedUnimplementedMessageServiceServer() +} + +// UnimplementedMessageServiceServer must be embedded to have forward compatible implementations. +type UnimplementedMessageServiceServer struct { +} + +func (UnimplementedMessageServiceServer) Connect(MessageService_ConnectServer) error { + return status.Errorf(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedMessageServiceServer) mustEmbedUnimplementedMessageServiceServer() {} + +// UnsafeMessageServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to MessageServiceServer will +// result in compilation errors. +type UnsafeMessageServiceServer interface { + mustEmbedUnimplementedMessageServiceServer() +} + +func RegisterMessageServiceServer(s grpc.ServiceRegistrar, srv MessageServiceServer) { + s.RegisterService(&MessageService_ServiceDesc, srv) +} + +func _MessageService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(MessageServiceServer).Connect(&messageServiceConnectServer{stream}) +} + +type MessageService_ConnectServer interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ServerStream +} + +type messageServiceConnectServer struct { + grpc.ServerStream +} + +func (x *messageServiceConnectServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func (x *messageServiceConnectServer) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MessageService_ServiceDesc is the grpc.ServiceDesc for MessageService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var MessageService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.MessageService", + HandlerType: (*MessageServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _MessageService_Connect_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/message_service.proto", +} diff --git a/grpc/model_base_service.pb.go b/grpc/model_base_service.pb.go new file mode 100644 index 00000000..0f9af695 --- /dev/null +++ b/grpc/model_base_service.pb.go @@ -0,0 +1,134 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/model_base_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_model_base_service_proto protoreflect.FileDescriptor + +var file_services_model_base_service_proto_rawDesc = []byte{ + 0x0a, 0x21, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xac, 0x04, 0x0a, 0x10, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x42, 0x61, 0x73, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2a, 0x0a, 0x07, 0x47, + 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x26, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x0d, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, + 0x2a, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x0a, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x29, 0x0a, 0x06, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4c, + 0x69, 0x73, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x32, 0x0a, 0x0f, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x2d, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x29, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x2c, 0x0a, 0x09, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x63, 0x12, + 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, + 0x12, 0x29, 0x0a, 0x06, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x28, 0x0a, 0x05, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_model_base_service_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request + (*Response)(nil), // 1: grpc.Response +} +var file_services_model_base_service_proto_depIdxs = []int32{ + 0, // 0: grpc.ModelBaseService.GetById:input_type -> grpc.Request + 0, // 1: grpc.ModelBaseService.Get:input_type -> grpc.Request + 0, // 2: grpc.ModelBaseService.GetList:input_type -> grpc.Request + 0, // 3: grpc.ModelBaseService.DeleteById:input_type -> grpc.Request + 0, // 4: grpc.ModelBaseService.Delete:input_type -> grpc.Request + 0, // 5: grpc.ModelBaseService.DeleteList:input_type -> grpc.Request + 0, // 6: grpc.ModelBaseService.ForceDeleteList:input_type -> grpc.Request + 0, // 7: grpc.ModelBaseService.UpdateById:input_type -> grpc.Request + 0, // 8: grpc.ModelBaseService.Update:input_type -> grpc.Request + 0, // 9: grpc.ModelBaseService.UpdateDoc:input_type -> grpc.Request + 0, // 10: grpc.ModelBaseService.Insert:input_type -> grpc.Request + 0, // 11: grpc.ModelBaseService.Count:input_type -> grpc.Request + 1, // 12: grpc.ModelBaseService.GetById:output_type -> grpc.Response + 1, // 13: grpc.ModelBaseService.Get:output_type -> grpc.Response + 1, // 14: grpc.ModelBaseService.GetList:output_type -> grpc.Response + 1, // 15: grpc.ModelBaseService.DeleteById:output_type -> grpc.Response + 1, // 16: grpc.ModelBaseService.Delete:output_type -> grpc.Response + 1, // 17: grpc.ModelBaseService.DeleteList:output_type -> grpc.Response + 1, // 18: grpc.ModelBaseService.ForceDeleteList:output_type -> grpc.Response + 1, // 19: grpc.ModelBaseService.UpdateById:output_type -> grpc.Response + 1, // 20: grpc.ModelBaseService.Update:output_type -> grpc.Response + 1, // 21: grpc.ModelBaseService.UpdateDoc:output_type -> grpc.Response + 1, // 22: grpc.ModelBaseService.Insert:output_type -> grpc.Response + 1, // 23: grpc.ModelBaseService.Count:output_type -> grpc.Response + 12, // [12:24] is the sub-list for method output_type + 0, // [0:12] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_model_base_service_proto_init() } +func file_services_model_base_service_proto_init() { + if File_services_model_base_service_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_model_base_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_model_base_service_proto_goTypes, + DependencyIndexes: file_services_model_base_service_proto_depIdxs, + }.Build() + File_services_model_base_service_proto = out.File + file_services_model_base_service_proto_rawDesc = nil + file_services_model_base_service_proto_goTypes = nil + file_services_model_base_service_proto_depIdxs = nil +} diff --git a/grpc/model_base_service_grpc.pb.go b/grpc/model_base_service_grpc.pb.go new file mode 100644 index 00000000..465355ec --- /dev/null +++ b/grpc/model_base_service_grpc.pb.go @@ -0,0 +1,501 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/model_base_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ModelBaseServiceClient is the client API for ModelBaseService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ModelBaseServiceClient interface { + GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type modelBaseServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewModelBaseServiceClient(cc grpc.ClientConnInterface) ModelBaseServiceClient { + return &modelBaseServiceClient{cc} +} + +func (c *modelBaseServiceClient) GetById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Get", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) GetList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/GetList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) DeleteById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Delete", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) DeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/DeleteList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) ForceDeleteList(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/ForceDeleteList", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) UpdateById(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Update(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Update", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) UpdateDoc(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/UpdateDoc", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Insert(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Insert", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceClient) Count(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseService/Count", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ModelBaseServiceServer is the server API for ModelBaseService service. +// All implementations must embed UnimplementedModelBaseServiceServer +// for forward compatibility +type ModelBaseServiceServer interface { + GetById(context.Context, *Request) (*Response, error) + Get(context.Context, *Request) (*Response, error) + GetList(context.Context, *Request) (*Response, error) + DeleteById(context.Context, *Request) (*Response, error) + Delete(context.Context, *Request) (*Response, error) + DeleteList(context.Context, *Request) (*Response, error) + ForceDeleteList(context.Context, *Request) (*Response, error) + UpdateById(context.Context, *Request) (*Response, error) + Update(context.Context, *Request) (*Response, error) + UpdateDoc(context.Context, *Request) (*Response, error) + Insert(context.Context, *Request) (*Response, error) + Count(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedModelBaseServiceServer() +} + +// UnimplementedModelBaseServiceServer must be embedded to have forward compatible implementations. +type UnimplementedModelBaseServiceServer struct { +} + +func (UnimplementedModelBaseServiceServer) GetById(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetById not implemented") +} +func (UnimplementedModelBaseServiceServer) Get(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") +} +func (UnimplementedModelBaseServiceServer) GetList(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetList not implemented") +} +func (UnimplementedModelBaseServiceServer) DeleteById(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteById not implemented") +} +func (UnimplementedModelBaseServiceServer) Delete(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") +} +func (UnimplementedModelBaseServiceServer) DeleteList(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteList not implemented") +} +func (UnimplementedModelBaseServiceServer) ForceDeleteList(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ForceDeleteList not implemented") +} +func (UnimplementedModelBaseServiceServer) UpdateById(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateById not implemented") +} +func (UnimplementedModelBaseServiceServer) Update(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Update not implemented") +} +func (UnimplementedModelBaseServiceServer) UpdateDoc(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateDoc not implemented") +} +func (UnimplementedModelBaseServiceServer) Insert(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Insert not implemented") +} +func (UnimplementedModelBaseServiceServer) Count(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Count not implemented") +} +func (UnimplementedModelBaseServiceServer) mustEmbedUnimplementedModelBaseServiceServer() {} + +// UnsafeModelBaseServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ModelBaseServiceServer will +// result in compilation errors. +type UnsafeModelBaseServiceServer interface { + mustEmbedUnimplementedModelBaseServiceServer() +} + +func RegisterModelBaseServiceServer(s grpc.ServiceRegistrar, srv ModelBaseServiceServer) { + s.RegisterService(&ModelBaseService_ServiceDesc, srv) +} + +func _ModelBaseService_GetById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).GetById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/GetById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).GetById(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Get(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Get", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Get(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_GetList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).GetList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/GetList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).GetList(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_DeleteById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).DeleteById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/DeleteById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).DeleteById(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Delete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Delete", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Delete(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_DeleteList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).DeleteList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/DeleteList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).DeleteList(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_ForceDeleteList_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/ForceDeleteList", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).ForceDeleteList(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_UpdateById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).UpdateById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/UpdateById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).UpdateById(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Update(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Update", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Update(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_UpdateDoc_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).UpdateDoc(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/UpdateDoc", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).UpdateDoc(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Insert_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Insert(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Insert", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Insert(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseService_Count_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceServer).Count(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseService/Count", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceServer).Count(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// ModelBaseService_ServiceDesc is the grpc.ServiceDesc for ModelBaseService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ModelBaseService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.ModelBaseService", + HandlerType: (*ModelBaseServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetById", + Handler: _ModelBaseService_GetById_Handler, + }, + { + MethodName: "Get", + Handler: _ModelBaseService_Get_Handler, + }, + { + MethodName: "GetList", + Handler: _ModelBaseService_GetList_Handler, + }, + { + MethodName: "DeleteById", + Handler: _ModelBaseService_DeleteById_Handler, + }, + { + MethodName: "Delete", + Handler: _ModelBaseService_Delete_Handler, + }, + { + MethodName: "DeleteList", + Handler: _ModelBaseService_DeleteList_Handler, + }, + { + MethodName: "ForceDeleteList", + Handler: _ModelBaseService_ForceDeleteList_Handler, + }, + { + MethodName: "UpdateById", + Handler: _ModelBaseService_UpdateById_Handler, + }, + { + MethodName: "Update", + Handler: _ModelBaseService_Update_Handler, + }, + { + MethodName: "UpdateDoc", + Handler: _ModelBaseService_UpdateDoc_Handler, + }, + { + MethodName: "Insert", + Handler: _ModelBaseService_Insert_Handler, + }, + { + MethodName: "Count", + Handler: _ModelBaseService_Count_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "services/model_base_service.proto", +} diff --git a/grpc/model_base_service_v2.pb.go b/grpc/model_base_service_v2.pb.go new file mode 100644 index 00000000..22377ca9 --- /dev/null +++ b/grpc/model_base_service_v2.pb.go @@ -0,0 +1,179 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/model_base_service_v2.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_model_base_service_v2_proto protoreflect.FileDescriptor + +var file_services_model_base_service_v2_proto_rawDesc = []byte{ + 0x0a, 0x24, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x5f, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x25, 0x65, 0x6e, + 0x74, 0x69, 0x74, 0x79, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xd4, 0x07, 0x0a, 0x12, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x42, 0x61, 0x73, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x12, 0x3f, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x12, 0x22, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x47, 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x65, 0x12, 0x21, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x3f, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x12, 0x22, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, + 0x12, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, + 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x09, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, + 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, 0x65, 0x6c, 0x65, + 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, + 0x0a, 0x0a, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x12, 0x25, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x56, 0x32, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, + 0x79, 0x49, 0x64, 0x12, 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, + 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x09, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x12, + 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, + 0x61, 0x63, 0x65, 0x42, 0x79, 0x49, 0x64, 0x12, 0x26, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x52, 0x65, 0x70, + 0x6c, 0x61, 0x63, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x45, 0x0a, 0x0a, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x65, 0x12, + 0x25, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x09, 0x49, 0x6e, 0x73, 0x65, + 0x72, 0x74, 0x4f, 0x6e, 0x65, 0x12, 0x24, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, + 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, 0x73, 0x65, 0x72, + 0x74, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x45, 0x0a, + 0x0a, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x12, 0x25, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, + 0x32, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x00, 0x12, 0x3b, 0x0a, 0x05, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x20, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x56, 0x32, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var file_services_model_base_service_v2_proto_goTypes = []interface{}{ + (*ModelServiceV2GetByIdRequest)(nil), // 0: grpc.ModelServiceV2GetByIdRequest + (*ModelServiceV2GetOneRequest)(nil), // 1: grpc.ModelServiceV2GetOneRequest + (*ModelServiceV2GetManyRequest)(nil), // 2: grpc.ModelServiceV2GetManyRequest + (*ModelServiceV2DeleteByIdRequest)(nil), // 3: grpc.ModelServiceV2DeleteByIdRequest + (*ModelServiceV2DeleteOneRequest)(nil), // 4: grpc.ModelServiceV2DeleteOneRequest + (*ModelServiceV2DeleteManyRequest)(nil), // 5: grpc.ModelServiceV2DeleteManyRequest + (*ModelServiceV2UpdateByIdRequest)(nil), // 6: grpc.ModelServiceV2UpdateByIdRequest + (*ModelServiceV2UpdateOneRequest)(nil), // 7: grpc.ModelServiceV2UpdateOneRequest + (*ModelServiceV2UpdateManyRequest)(nil), // 8: grpc.ModelServiceV2UpdateManyRequest + (*ModelServiceV2ReplaceByIdRequest)(nil), // 9: grpc.ModelServiceV2ReplaceByIdRequest + (*ModelServiceV2ReplaceOneRequest)(nil), // 10: grpc.ModelServiceV2ReplaceOneRequest + (*ModelServiceV2InsertOneRequest)(nil), // 11: grpc.ModelServiceV2InsertOneRequest + (*ModelServiceV2InsertManyRequest)(nil), // 12: grpc.ModelServiceV2InsertManyRequest + (*ModelServiceV2CountRequest)(nil), // 13: grpc.ModelServiceV2CountRequest + (*Response)(nil), // 14: grpc.Response +} +var file_services_model_base_service_v2_proto_depIdxs = []int32{ + 0, // 0: grpc.ModelBaseServiceV2.GetById:input_type -> grpc.ModelServiceV2GetByIdRequest + 1, // 1: grpc.ModelBaseServiceV2.GetOne:input_type -> grpc.ModelServiceV2GetOneRequest + 2, // 2: grpc.ModelBaseServiceV2.GetMany:input_type -> grpc.ModelServiceV2GetManyRequest + 3, // 3: grpc.ModelBaseServiceV2.DeleteById:input_type -> grpc.ModelServiceV2DeleteByIdRequest + 4, // 4: grpc.ModelBaseServiceV2.DeleteOne:input_type -> grpc.ModelServiceV2DeleteOneRequest + 5, // 5: grpc.ModelBaseServiceV2.DeleteMany:input_type -> grpc.ModelServiceV2DeleteManyRequest + 6, // 6: grpc.ModelBaseServiceV2.UpdateById:input_type -> grpc.ModelServiceV2UpdateByIdRequest + 7, // 7: grpc.ModelBaseServiceV2.UpdateOne:input_type -> grpc.ModelServiceV2UpdateOneRequest + 8, // 8: grpc.ModelBaseServiceV2.UpdateMany:input_type -> grpc.ModelServiceV2UpdateManyRequest + 9, // 9: grpc.ModelBaseServiceV2.ReplaceById:input_type -> grpc.ModelServiceV2ReplaceByIdRequest + 10, // 10: grpc.ModelBaseServiceV2.ReplaceOne:input_type -> grpc.ModelServiceV2ReplaceOneRequest + 11, // 11: grpc.ModelBaseServiceV2.InsertOne:input_type -> grpc.ModelServiceV2InsertOneRequest + 12, // 12: grpc.ModelBaseServiceV2.InsertMany:input_type -> grpc.ModelServiceV2InsertManyRequest + 13, // 13: grpc.ModelBaseServiceV2.Count:input_type -> grpc.ModelServiceV2CountRequest + 14, // 14: grpc.ModelBaseServiceV2.GetById:output_type -> grpc.Response + 14, // 15: grpc.ModelBaseServiceV2.GetOne:output_type -> grpc.Response + 14, // 16: grpc.ModelBaseServiceV2.GetMany:output_type -> grpc.Response + 14, // 17: grpc.ModelBaseServiceV2.DeleteById:output_type -> grpc.Response + 14, // 18: grpc.ModelBaseServiceV2.DeleteOne:output_type -> grpc.Response + 14, // 19: grpc.ModelBaseServiceV2.DeleteMany:output_type -> grpc.Response + 14, // 20: grpc.ModelBaseServiceV2.UpdateById:output_type -> grpc.Response + 14, // 21: grpc.ModelBaseServiceV2.UpdateOne:output_type -> grpc.Response + 14, // 22: grpc.ModelBaseServiceV2.UpdateMany:output_type -> grpc.Response + 14, // 23: grpc.ModelBaseServiceV2.ReplaceById:output_type -> grpc.Response + 14, // 24: grpc.ModelBaseServiceV2.ReplaceOne:output_type -> grpc.Response + 14, // 25: grpc.ModelBaseServiceV2.InsertOne:output_type -> grpc.Response + 14, // 26: grpc.ModelBaseServiceV2.InsertMany:output_type -> grpc.Response + 14, // 27: grpc.ModelBaseServiceV2.Count:output_type -> grpc.Response + 14, // [14:28] is the sub-list for method output_type + 0, // [0:14] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_model_base_service_v2_proto_init() } +func file_services_model_base_service_v2_proto_init() { + if File_services_model_base_service_v2_proto != nil { + return + } + file_entity_model_service_v2_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_model_base_service_v2_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_model_base_service_v2_proto_goTypes, + DependencyIndexes: file_services_model_base_service_v2_proto_depIdxs, + }.Build() + File_services_model_base_service_v2_proto = out.File + file_services_model_base_service_v2_proto_rawDesc = nil + file_services_model_base_service_v2_proto_goTypes = nil + file_services_model_base_service_v2_proto_depIdxs = nil +} diff --git a/grpc/model_base_service_v2_grpc.pb.go b/grpc/model_base_service_v2_grpc.pb.go new file mode 100644 index 00000000..6f5fac73 --- /dev/null +++ b/grpc/model_base_service_v2_grpc.pb.go @@ -0,0 +1,573 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/model_base_service_v2.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ModelBaseServiceV2Client is the client API for ModelBaseServiceV2 service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ModelBaseServiceV2Client interface { + GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) + GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) + GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) + DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) + DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) + DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) + UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) + UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) + UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) + ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) + ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) + InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) + InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) + Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) +} + +type modelBaseServiceV2Client struct { + cc grpc.ClientConnInterface +} + +func NewModelBaseServiceV2Client(cc grpc.ClientConnInterface) ModelBaseServiceV2Client { + return &modelBaseServiceV2Client{cc} +} + +func (c *modelBaseServiceV2Client) GetById(ctx context.Context, in *ModelServiceV2GetByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) GetOne(ctx context.Context, in *ModelServiceV2GetOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) GetMany(ctx context.Context, in *ModelServiceV2GetManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/GetMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) DeleteById(ctx context.Context, in *ModelServiceV2DeleteByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) DeleteOne(ctx context.Context, in *ModelServiceV2DeleteOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) DeleteMany(ctx context.Context, in *ModelServiceV2DeleteManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/DeleteMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) UpdateById(ctx context.Context, in *ModelServiceV2UpdateByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) UpdateOne(ctx context.Context, in *ModelServiceV2UpdateOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) UpdateMany(ctx context.Context, in *ModelServiceV2UpdateManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/UpdateMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) ReplaceById(ctx context.Context, in *ModelServiceV2ReplaceByIdRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceById", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) ReplaceOne(ctx context.Context, in *ModelServiceV2ReplaceOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/ReplaceOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) InsertOne(ctx context.Context, in *ModelServiceV2InsertOneRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertOne", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) InsertMany(ctx context.Context, in *ModelServiceV2InsertManyRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/InsertMany", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *modelBaseServiceV2Client) Count(ctx context.Context, in *ModelServiceV2CountRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelBaseServiceV2/Count", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ModelBaseServiceV2Server is the server API for ModelBaseServiceV2 service. +// All implementations must embed UnimplementedModelBaseServiceV2Server +// for forward compatibility +type ModelBaseServiceV2Server interface { + GetById(context.Context, *ModelServiceV2GetByIdRequest) (*Response, error) + GetOne(context.Context, *ModelServiceV2GetOneRequest) (*Response, error) + GetMany(context.Context, *ModelServiceV2GetManyRequest) (*Response, error) + DeleteById(context.Context, *ModelServiceV2DeleteByIdRequest) (*Response, error) + DeleteOne(context.Context, *ModelServiceV2DeleteOneRequest) (*Response, error) + DeleteMany(context.Context, *ModelServiceV2DeleteManyRequest) (*Response, error) + UpdateById(context.Context, *ModelServiceV2UpdateByIdRequest) (*Response, error) + UpdateOne(context.Context, *ModelServiceV2UpdateOneRequest) (*Response, error) + UpdateMany(context.Context, *ModelServiceV2UpdateManyRequest) (*Response, error) + ReplaceById(context.Context, *ModelServiceV2ReplaceByIdRequest) (*Response, error) + ReplaceOne(context.Context, *ModelServiceV2ReplaceOneRequest) (*Response, error) + InsertOne(context.Context, *ModelServiceV2InsertOneRequest) (*Response, error) + InsertMany(context.Context, *ModelServiceV2InsertManyRequest) (*Response, error) + Count(context.Context, *ModelServiceV2CountRequest) (*Response, error) + mustEmbedUnimplementedModelBaseServiceV2Server() +} + +// UnimplementedModelBaseServiceV2Server must be embedded to have forward compatible implementations. +type UnimplementedModelBaseServiceV2Server struct { +} + +func (UnimplementedModelBaseServiceV2Server) GetById(context.Context, *ModelServiceV2GetByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) GetOne(context.Context, *ModelServiceV2GetOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) GetMany(context.Context, *ModelServiceV2GetManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) DeleteById(context.Context, *ModelServiceV2DeleteByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) DeleteOne(context.Context, *ModelServiceV2DeleteOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) DeleteMany(context.Context, *ModelServiceV2DeleteManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) UpdateById(context.Context, *ModelServiceV2UpdateByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) UpdateOne(context.Context, *ModelServiceV2UpdateOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) UpdateMany(context.Context, *ModelServiceV2UpdateManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) ReplaceById(context.Context, *ModelServiceV2ReplaceByIdRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceById not implemented") +} +func (UnimplementedModelBaseServiceV2Server) ReplaceOne(context.Context, *ModelServiceV2ReplaceOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) InsertOne(context.Context, *ModelServiceV2InsertOneRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method InsertOne not implemented") +} +func (UnimplementedModelBaseServiceV2Server) InsertMany(context.Context, *ModelServiceV2InsertManyRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method InsertMany not implemented") +} +func (UnimplementedModelBaseServiceV2Server) Count(context.Context, *ModelServiceV2CountRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Count not implemented") +} +func (UnimplementedModelBaseServiceV2Server) mustEmbedUnimplementedModelBaseServiceV2Server() {} + +// UnsafeModelBaseServiceV2Server may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ModelBaseServiceV2Server will +// result in compilation errors. +type UnsafeModelBaseServiceV2Server interface { + mustEmbedUnimplementedModelBaseServiceV2Server() +} + +func RegisterModelBaseServiceV2Server(s grpc.ServiceRegistrar, srv ModelBaseServiceV2Server) { + s.RegisterService(&ModelBaseServiceV2_ServiceDesc, srv) +} + +func _ModelBaseServiceV2_GetById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2GetByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).GetById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/GetById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).GetById(ctx, req.(*ModelServiceV2GetByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_GetOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2GetOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).GetOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/GetOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).GetOne(ctx, req.(*ModelServiceV2GetOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_GetMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2GetManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).GetMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/GetMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).GetMany(ctx, req.(*ModelServiceV2GetManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_DeleteById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2DeleteByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).DeleteById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).DeleteById(ctx, req.(*ModelServiceV2DeleteByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_DeleteOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2DeleteOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).DeleteOne(ctx, req.(*ModelServiceV2DeleteOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_DeleteMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2DeleteManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/DeleteMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).DeleteMany(ctx, req.(*ModelServiceV2DeleteManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_UpdateById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2UpdateByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).UpdateById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).UpdateById(ctx, req.(*ModelServiceV2UpdateByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_UpdateOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2UpdateOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).UpdateOne(ctx, req.(*ModelServiceV2UpdateOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_UpdateMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2UpdateManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/UpdateMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).UpdateMany(ctx, req.(*ModelServiceV2UpdateManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_ReplaceById_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2ReplaceByIdRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/ReplaceById", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).ReplaceById(ctx, req.(*ModelServiceV2ReplaceByIdRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_ReplaceOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2ReplaceOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/ReplaceOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).ReplaceOne(ctx, req.(*ModelServiceV2ReplaceOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_InsertOne_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2InsertOneRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).InsertOne(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/InsertOne", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).InsertOne(ctx, req.(*ModelServiceV2InsertOneRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_InsertMany_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2InsertManyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).InsertMany(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/InsertMany", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).InsertMany(ctx, req.(*ModelServiceV2InsertManyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ModelBaseServiceV2_Count_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ModelServiceV2CountRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelBaseServiceV2Server).Count(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelBaseServiceV2/Count", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelBaseServiceV2Server).Count(ctx, req.(*ModelServiceV2CountRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ModelBaseServiceV2_ServiceDesc is the grpc.ServiceDesc for ModelBaseServiceV2 service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ModelBaseServiceV2_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.ModelBaseServiceV2", + HandlerType: (*ModelBaseServiceV2Server)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetById", + Handler: _ModelBaseServiceV2_GetById_Handler, + }, + { + MethodName: "GetOne", + Handler: _ModelBaseServiceV2_GetOne_Handler, + }, + { + MethodName: "GetMany", + Handler: _ModelBaseServiceV2_GetMany_Handler, + }, + { + MethodName: "DeleteById", + Handler: _ModelBaseServiceV2_DeleteById_Handler, + }, + { + MethodName: "DeleteOne", + Handler: _ModelBaseServiceV2_DeleteOne_Handler, + }, + { + MethodName: "DeleteMany", + Handler: _ModelBaseServiceV2_DeleteMany_Handler, + }, + { + MethodName: "UpdateById", + Handler: _ModelBaseServiceV2_UpdateById_Handler, + }, + { + MethodName: "UpdateOne", + Handler: _ModelBaseServiceV2_UpdateOne_Handler, + }, + { + MethodName: "UpdateMany", + Handler: _ModelBaseServiceV2_UpdateMany_Handler, + }, + { + MethodName: "ReplaceById", + Handler: _ModelBaseServiceV2_ReplaceById_Handler, + }, + { + MethodName: "ReplaceOne", + Handler: _ModelBaseServiceV2_ReplaceOne_Handler, + }, + { + MethodName: "InsertOne", + Handler: _ModelBaseServiceV2_InsertOne_Handler, + }, + { + MethodName: "InsertMany", + Handler: _ModelBaseServiceV2_InsertMany_Handler, + }, + { + MethodName: "Count", + Handler: _ModelBaseServiceV2_Count_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "services/model_base_service_v2.proto", +} diff --git a/grpc/model_delegate.pb.go b/grpc/model_delegate.pb.go new file mode 100644 index 00000000..76dfa489 --- /dev/null +++ b/grpc/model_delegate.pb.go @@ -0,0 +1,80 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/model_delegate.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_model_delegate_proto protoreflect.FileDescriptor + +var file_services_model_delegate_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, + 0x5f, 0x64, 0x65, 0x6c, 0x65, 0x67, 0x61, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, + 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x32, 0x36, 0x0a, 0x0d, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x12, 0x25, 0x0a, 0x02, 0x44, 0x6f, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_model_delegate_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request + (*Response)(nil), // 1: grpc.Response +} +var file_services_model_delegate_proto_depIdxs = []int32{ + 0, // 0: grpc.ModelDelegate.Do:input_type -> grpc.Request + 1, // 1: grpc.ModelDelegate.Do:output_type -> grpc.Response + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_model_delegate_proto_init() } +func file_services_model_delegate_proto_init() { + if File_services_model_delegate_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_model_delegate_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_model_delegate_proto_goTypes, + DependencyIndexes: file_services_model_delegate_proto_depIdxs, + }.Build() + File_services_model_delegate_proto = out.File + file_services_model_delegate_proto_rawDesc = nil + file_services_model_delegate_proto_goTypes = nil + file_services_model_delegate_proto_depIdxs = nil +} diff --git a/grpc/model_delegate_grpc.pb.go b/grpc/model_delegate_grpc.pb.go new file mode 100644 index 00000000..3ab1efa5 --- /dev/null +++ b/grpc/model_delegate_grpc.pb.go @@ -0,0 +1,105 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/model_delegate.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// ModelDelegateClient is the client API for ModelDelegate service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ModelDelegateClient interface { + Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type modelDelegateClient struct { + cc grpc.ClientConnInterface +} + +func NewModelDelegateClient(cc grpc.ClientConnInterface) ModelDelegateClient { + return &modelDelegateClient{cc} +} + +func (c *modelDelegateClient) Do(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.ModelDelegate/Do", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ModelDelegateServer is the server API for ModelDelegate service. +// All implementations must embed UnimplementedModelDelegateServer +// for forward compatibility +type ModelDelegateServer interface { + Do(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedModelDelegateServer() +} + +// UnimplementedModelDelegateServer must be embedded to have forward compatible implementations. +type UnimplementedModelDelegateServer struct { +} + +func (UnimplementedModelDelegateServer) Do(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Do not implemented") +} +func (UnimplementedModelDelegateServer) mustEmbedUnimplementedModelDelegateServer() {} + +// UnsafeModelDelegateServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ModelDelegateServer will +// result in compilation errors. +type UnsafeModelDelegateServer interface { + mustEmbedUnimplementedModelDelegateServer() +} + +func RegisterModelDelegateServer(s grpc.ServiceRegistrar, srv ModelDelegateServer) { + s.RegisterService(&ModelDelegate_ServiceDesc, srv) +} + +func _ModelDelegate_Do_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ModelDelegateServer).Do(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.ModelDelegate/Do", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ModelDelegateServer).Do(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// ModelDelegate_ServiceDesc is the grpc.ServiceDesc for ModelDelegate service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ModelDelegate_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.ModelDelegate", + HandlerType: (*ModelDelegateServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Do", + Handler: _ModelDelegate_Do_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "services/model_delegate.proto", +} diff --git a/grpc/model_service_v2_request.pb.go b/grpc/model_service_v2_request.pb.go new file mode 100644 index 00000000..b8958a9c --- /dev/null +++ b/grpc/model_service_v2_request.pb.go @@ -0,0 +1,1316 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/model_service_v2_request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type ModelServiceV2GetByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *ModelServiceV2GetByIdRequest) Reset() { + *x = ModelServiceV2GetByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2GetByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2GetByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2GetByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2GetByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2GetByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{0} +} + +func (x *ModelServiceV2GetByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2GetByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2GetByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ModelServiceV2GetOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + FindOptions []byte `protobuf:"bytes,4,opt,name=find_options,json=findOptions,proto3" json:"find_options,omitempty"` +} + +func (x *ModelServiceV2GetOneRequest) Reset() { + *x = ModelServiceV2GetOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2GetOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2GetOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2GetOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2GetOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2GetOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{1} +} + +func (x *ModelServiceV2GetOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2GetOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2GetOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2GetOneRequest) GetFindOptions() []byte { + if x != nil { + return x.FindOptions + } + return nil +} + +type ModelServiceV2GetManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + FindOptions []byte `protobuf:"bytes,4,opt,name=find_options,json=findOptions,proto3" json:"find_options,omitempty"` +} + +func (x *ModelServiceV2GetManyRequest) Reset() { + *x = ModelServiceV2GetManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2GetManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2GetManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2GetManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2GetManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2GetManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{2} +} + +func (x *ModelServiceV2GetManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2GetManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2GetManyRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2GetManyRequest) GetFindOptions() []byte { + if x != nil { + return x.FindOptions + } + return nil +} + +type ModelServiceV2DeleteByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *ModelServiceV2DeleteByIdRequest) Reset() { + *x = ModelServiceV2DeleteByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2DeleteByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2DeleteByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2DeleteByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2DeleteByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2DeleteByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{3} +} + +func (x *ModelServiceV2DeleteByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2DeleteByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2DeleteByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +type ModelServiceV2DeleteOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *ModelServiceV2DeleteOneRequest) Reset() { + *x = ModelServiceV2DeleteOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2DeleteOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2DeleteOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2DeleteOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2DeleteOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2DeleteOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{4} +} + +func (x *ModelServiceV2DeleteOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2DeleteOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2DeleteOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +type ModelServiceV2DeleteManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *ModelServiceV2DeleteManyRequest) Reset() { + *x = ModelServiceV2DeleteManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2DeleteManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2DeleteManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2DeleteManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2DeleteManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2DeleteManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{5} +} + +func (x *ModelServiceV2DeleteManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2DeleteManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2DeleteManyRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +type ModelServiceV2UpdateByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + Update []byte `protobuf:"bytes,4,opt,name=update,proto3" json:"update,omitempty"` +} + +func (x *ModelServiceV2UpdateByIdRequest) Reset() { + *x = ModelServiceV2UpdateByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2UpdateByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2UpdateByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2UpdateByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2UpdateByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2UpdateByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{6} +} + +func (x *ModelServiceV2UpdateByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2UpdateByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2UpdateByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ModelServiceV2UpdateByIdRequest) GetUpdate() []byte { + if x != nil { + return x.Update + } + return nil +} + +type ModelServiceV2UpdateOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Update []byte `protobuf:"bytes,4,opt,name=update,proto3" json:"update,omitempty"` +} + +func (x *ModelServiceV2UpdateOneRequest) Reset() { + *x = ModelServiceV2UpdateOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2UpdateOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2UpdateOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2UpdateOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2UpdateOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2UpdateOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{7} +} + +func (x *ModelServiceV2UpdateOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2UpdateOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2UpdateOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2UpdateOneRequest) GetUpdate() []byte { + if x != nil { + return x.Update + } + return nil +} + +type ModelServiceV2UpdateManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Update []byte `protobuf:"bytes,4,opt,name=update,proto3" json:"update,omitempty"` +} + +func (x *ModelServiceV2UpdateManyRequest) Reset() { + *x = ModelServiceV2UpdateManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2UpdateManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2UpdateManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2UpdateManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2UpdateManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2UpdateManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{8} +} + +func (x *ModelServiceV2UpdateManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2UpdateManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2UpdateManyRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2UpdateManyRequest) GetUpdate() []byte { + if x != nil { + return x.Update + } + return nil +} + +type ModelServiceV2ReplaceByIdRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Id string `protobuf:"bytes,3,opt,name=id,proto3" json:"id,omitempty"` + Model []byte `protobuf:"bytes,4,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *ModelServiceV2ReplaceByIdRequest) Reset() { + *x = ModelServiceV2ReplaceByIdRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2ReplaceByIdRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2ReplaceByIdRequest) ProtoMessage() {} + +func (x *ModelServiceV2ReplaceByIdRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2ReplaceByIdRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2ReplaceByIdRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{9} +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *ModelServiceV2ReplaceByIdRequest) GetModel() []byte { + if x != nil { + return x.Model + } + return nil +} + +type ModelServiceV2ReplaceOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` + Model []byte `protobuf:"bytes,4,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *ModelServiceV2ReplaceOneRequest) Reset() { + *x = ModelServiceV2ReplaceOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2ReplaceOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2ReplaceOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2ReplaceOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2ReplaceOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2ReplaceOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{10} +} + +func (x *ModelServiceV2ReplaceOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2ReplaceOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2ReplaceOneRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +func (x *ModelServiceV2ReplaceOneRequest) GetModel() []byte { + if x != nil { + return x.Model + } + return nil +} + +type ModelServiceV2InsertOneRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Model []byte `protobuf:"bytes,3,opt,name=model,proto3" json:"model,omitempty"` +} + +func (x *ModelServiceV2InsertOneRequest) Reset() { + *x = ModelServiceV2InsertOneRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2InsertOneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2InsertOneRequest) ProtoMessage() {} + +func (x *ModelServiceV2InsertOneRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2InsertOneRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2InsertOneRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{11} +} + +func (x *ModelServiceV2InsertOneRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2InsertOneRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2InsertOneRequest) GetModel() []byte { + if x != nil { + return x.Model + } + return nil +} + +type ModelServiceV2InsertManyRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Models []byte `protobuf:"bytes,3,opt,name=models,proto3" json:"models,omitempty"` +} + +func (x *ModelServiceV2InsertManyRequest) Reset() { + *x = ModelServiceV2InsertManyRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2InsertManyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2InsertManyRequest) ProtoMessage() {} + +func (x *ModelServiceV2InsertManyRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2InsertManyRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2InsertManyRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{12} +} + +func (x *ModelServiceV2InsertManyRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2InsertManyRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2InsertManyRequest) GetModels() []byte { + if x != nil { + return x.Models + } + return nil +} + +type ModelServiceV2CountRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + ModelType string `protobuf:"bytes,2,opt,name=model_type,json=modelType,proto3" json:"model_type,omitempty"` + Query []byte `protobuf:"bytes,3,opt,name=query,proto3" json:"query,omitempty"` +} + +func (x *ModelServiceV2CountRequest) Reset() { + *x = ModelServiceV2CountRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_model_service_v2_request_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ModelServiceV2CountRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ModelServiceV2CountRequest) ProtoMessage() {} + +func (x *ModelServiceV2CountRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_model_service_v2_request_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ModelServiceV2CountRequest.ProtoReflect.Descriptor instead. +func (*ModelServiceV2CountRequest) Descriptor() ([]byte, []int) { + return file_entity_model_service_v2_request_proto_rawDescGZIP(), []int{13} +} + +func (x *ModelServiceV2CountRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *ModelServiceV2CountRequest) GetModelType() string { + if x != nil { + return x.ModelType + } + return "" +} + +func (x *ModelServiceV2CountRequest) GetQuery() []byte { + if x != nil { + return x.Query + } + return nil +} + +var File_entity_model_service_v2_request_proto protoreflect.FileDescriptor + +var file_entity_model_service_v2_request_proto_rawDesc = []byte{ + 0x0a, 0x25, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x73, + 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5f, 0x76, 0x32, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x68, 0x0a, + 0x1c, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x47, + 0x65, 0x74, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x90, 0x01, 0x0a, 0x1b, 0x4d, 0x6f, 0x64, 0x65, + 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x47, 0x65, 0x74, 0x4f, 0x6e, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, + 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x66, 0x69, 0x6e, 0x64, 0x5f, + 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x66, + 0x69, 0x6e, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x91, 0x01, 0x0a, 0x1c, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x47, 0x65, 0x74, + 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, + 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, + 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x66, + 0x69, 0x6e, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x0b, 0x66, 0x69, 0x6e, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x6b, + 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x70, 0x0a, 0x1e, 0x4d, + 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, + 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, + 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x71, 0x0a, + 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, + 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, + 0x22, 0x83, 0x01, 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, + 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x88, 0x01, 0x0a, 0x1e, 0x4d, 0x6f, 0x64, 0x65, 0x6c, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4f, + 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, + 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, + 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x22, 0x89, 0x01, 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x56, 0x32, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, + 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, + 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x22, 0x82, 0x01, + 0x0a, 0x20, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x42, 0x79, 0x49, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, + 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x22, 0x87, 0x01, 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x6e, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, + 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x14, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x70, 0x0a, 0x1e, + 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, 0x49, 0x6e, + 0x73, 0x65, 0x72, 0x74, 0x4f, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, + 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x6f, 0x64, 0x65, + 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x22, 0x73, + 0x0a, 0x1f, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x56, 0x32, + 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x4d, 0x61, 0x6e, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6d, + 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6d, 0x6f, 0x64, + 0x65, 0x6c, 0x73, 0x22, 0x6c, 0x0a, 0x1a, 0x4d, 0x6f, 0x64, 0x65, 0x6c, 0x53, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x56, 0x32, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, + 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x71, + 0x75, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, + 0x79, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_entity_model_service_v2_request_proto_rawDescOnce sync.Once + file_entity_model_service_v2_request_proto_rawDescData = file_entity_model_service_v2_request_proto_rawDesc +) + +func file_entity_model_service_v2_request_proto_rawDescGZIP() []byte { + file_entity_model_service_v2_request_proto_rawDescOnce.Do(func() { + file_entity_model_service_v2_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_model_service_v2_request_proto_rawDescData) + }) + return file_entity_model_service_v2_request_proto_rawDescData +} + +var file_entity_model_service_v2_request_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_entity_model_service_v2_request_proto_goTypes = []interface{}{ + (*ModelServiceV2GetByIdRequest)(nil), // 0: grpc.ModelServiceV2GetByIdRequest + (*ModelServiceV2GetOneRequest)(nil), // 1: grpc.ModelServiceV2GetOneRequest + (*ModelServiceV2GetManyRequest)(nil), // 2: grpc.ModelServiceV2GetManyRequest + (*ModelServiceV2DeleteByIdRequest)(nil), // 3: grpc.ModelServiceV2DeleteByIdRequest + (*ModelServiceV2DeleteOneRequest)(nil), // 4: grpc.ModelServiceV2DeleteOneRequest + (*ModelServiceV2DeleteManyRequest)(nil), // 5: grpc.ModelServiceV2DeleteManyRequest + (*ModelServiceV2UpdateByIdRequest)(nil), // 6: grpc.ModelServiceV2UpdateByIdRequest + (*ModelServiceV2UpdateOneRequest)(nil), // 7: grpc.ModelServiceV2UpdateOneRequest + (*ModelServiceV2UpdateManyRequest)(nil), // 8: grpc.ModelServiceV2UpdateManyRequest + (*ModelServiceV2ReplaceByIdRequest)(nil), // 9: grpc.ModelServiceV2ReplaceByIdRequest + (*ModelServiceV2ReplaceOneRequest)(nil), // 10: grpc.ModelServiceV2ReplaceOneRequest + (*ModelServiceV2InsertOneRequest)(nil), // 11: grpc.ModelServiceV2InsertOneRequest + (*ModelServiceV2InsertManyRequest)(nil), // 12: grpc.ModelServiceV2InsertManyRequest + (*ModelServiceV2CountRequest)(nil), // 13: grpc.ModelServiceV2CountRequest +} +var file_entity_model_service_v2_request_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_model_service_v2_request_proto_init() } +func file_entity_model_service_v2_request_proto_init() { + if File_entity_model_service_v2_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_model_service_v2_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2GetByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2GetOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2GetManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2DeleteByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2DeleteOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2DeleteManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2UpdateByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2UpdateOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2UpdateManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2ReplaceByIdRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2ReplaceOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2InsertOneRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2InsertManyRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_entity_model_service_v2_request_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ModelServiceV2CountRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_model_service_v2_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 14, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_model_service_v2_request_proto_goTypes, + DependencyIndexes: file_entity_model_service_v2_request_proto_depIdxs, + MessageInfos: file_entity_model_service_v2_request_proto_msgTypes, + }.Build() + File_entity_model_service_v2_request_proto = out.File + file_entity_model_service_v2_request_proto_rawDesc = nil + file_entity_model_service_v2_request_proto_goTypes = nil + file_entity_model_service_v2_request_proto_depIdxs = nil +} diff --git a/grpc/node.pb.go b/grpc/node.pb.go new file mode 100644 index 00000000..2e32c400 --- /dev/null +++ b/grpc/node.pb.go @@ -0,0 +1,251 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: models/node.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Node struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + XId string `protobuf:"bytes,1,opt,name=_id,json=Id,proto3" json:"_id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Ip string `protobuf:"bytes,3,opt,name=ip,proto3" json:"ip,omitempty"` + Port string `protobuf:"bytes,5,opt,name=port,proto3" json:"port,omitempty"` + Mac string `protobuf:"bytes,6,opt,name=mac,proto3" json:"mac,omitempty"` + Hostname string `protobuf:"bytes,7,opt,name=hostname,proto3" json:"hostname,omitempty"` + Description string `protobuf:"bytes,8,opt,name=description,proto3" json:"description,omitempty"` + Key string `protobuf:"bytes,9,opt,name=key,proto3" json:"key,omitempty"` + IsMaster bool `protobuf:"varint,11,opt,name=is_master,json=isMaster,proto3" json:"is_master,omitempty"` + UpdateTs string `protobuf:"bytes,12,opt,name=update_ts,json=updateTs,proto3" json:"update_ts,omitempty"` + CreateTs string `protobuf:"bytes,13,opt,name=create_ts,json=createTs,proto3" json:"create_ts,omitempty"` + UpdateTsUnix int64 `protobuf:"varint,14,opt,name=update_ts_unix,json=updateTsUnix,proto3" json:"update_ts_unix,omitempty"` +} + +func (x *Node) Reset() { + *x = Node{} + if protoimpl.UnsafeEnabled { + mi := &file_models_node_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Node) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Node) ProtoMessage() {} + +func (x *Node) ProtoReflect() protoreflect.Message { + mi := &file_models_node_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Node.ProtoReflect.Descriptor instead. +func (*Node) Descriptor() ([]byte, []int) { + return file_models_node_proto_rawDescGZIP(), []int{0} +} + +func (x *Node) GetXId() string { + if x != nil { + return x.XId + } + return "" +} + +func (x *Node) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Node) GetIp() string { + if x != nil { + return x.Ip + } + return "" +} + +func (x *Node) GetPort() string { + if x != nil { + return x.Port + } + return "" +} + +func (x *Node) GetMac() string { + if x != nil { + return x.Mac + } + return "" +} + +func (x *Node) GetHostname() string { + if x != nil { + return x.Hostname + } + return "" +} + +func (x *Node) GetDescription() string { + if x != nil { + return x.Description + } + return "" +} + +func (x *Node) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *Node) GetIsMaster() bool { + if x != nil { + return x.IsMaster + } + return false +} + +func (x *Node) GetUpdateTs() string { + if x != nil { + return x.UpdateTs + } + return "" +} + +func (x *Node) GetCreateTs() string { + if x != nil { + return x.CreateTs + } + return "" +} + +func (x *Node) GetUpdateTsUnix() int64 { + if x != nil { + return x.UpdateTsUnix + } + return 0 +} + +var File_models_node_proto protoreflect.FileDescriptor + +var file_models_node_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0xae, 0x02, 0x0a, 0x04, 0x4e, 0x6f, + 0x64, 0x65, 0x12, 0x0f, 0x0a, 0x03, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x70, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x70, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x6f, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, + 0x61, 0x63, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6d, 0x61, 0x63, 0x12, 0x1a, 0x0a, + 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x08, 0x68, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, + 0x09, 0x69, 0x73, 0x5f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x08, 0x69, 0x73, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x73, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x54, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x5f, 0x74, 0x73, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x54, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x5f, 0x74, + 0x73, 0x5f, 0x75, 0x6e, 0x69, 0x78, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0c, 0x75, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x54, 0x73, 0x55, 0x6e, 0x69, 0x78, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_models_node_proto_rawDescOnce sync.Once + file_models_node_proto_rawDescData = file_models_node_proto_rawDesc +) + +func file_models_node_proto_rawDescGZIP() []byte { + file_models_node_proto_rawDescOnce.Do(func() { + file_models_node_proto_rawDescData = protoimpl.X.CompressGZIP(file_models_node_proto_rawDescData) + }) + return file_models_node_proto_rawDescData +} + +var file_models_node_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_models_node_proto_goTypes = []interface{}{ + (*Node)(nil), // 0: grpc.Node +} +var file_models_node_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_models_node_proto_init() } +func file_models_node_proto_init() { + if File_models_node_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_models_node_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Node); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_models_node_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_models_node_proto_goTypes, + DependencyIndexes: file_models_node_proto_depIdxs, + MessageInfos: file_models_node_proto_msgTypes, + }.Build() + File_models_node_proto = out.File + file_models_node_proto_rawDesc = nil + file_models_node_proto_goTypes = nil + file_models_node_proto_depIdxs = nil +} diff --git a/grpc/node_info.pb.go b/grpc/node_info.pb.go new file mode 100644 index 00000000..135ddcbf --- /dev/null +++ b/grpc/node_info.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/node_info.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type NodeInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + IsMaster bool `protobuf:"varint,2,opt,name=is_master,json=isMaster,proto3" json:"is_master,omitempty"` +} + +func (x *NodeInfo) Reset() { + *x = NodeInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_node_info_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *NodeInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeInfo) ProtoMessage() {} + +func (x *NodeInfo) ProtoReflect() protoreflect.Message { + mi := &file_entity_node_info_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeInfo.ProtoReflect.Descriptor instead. +func (*NodeInfo) Descriptor() ([]byte, []int) { + return file_entity_node_info_proto_rawDescGZIP(), []int{0} +} + +func (x *NodeInfo) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *NodeInfo) GetIsMaster() bool { + if x != nil { + return x.IsMaster + } + return false +} + +var File_entity_node_info_proto protoreflect.FileDescriptor + +var file_entity_node_info_proto_rawDesc = []byte{ + 0x0a, 0x16, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x6e, + 0x66, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x39, + 0x0a, 0x08, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1b, 0x0a, 0x09, + 0x69, 0x73, 0x5f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x08, 0x69, 0x73, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, + 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_node_info_proto_rawDescOnce sync.Once + file_entity_node_info_proto_rawDescData = file_entity_node_info_proto_rawDesc +) + +func file_entity_node_info_proto_rawDescGZIP() []byte { + file_entity_node_info_proto_rawDescOnce.Do(func() { + file_entity_node_info_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_node_info_proto_rawDescData) + }) + return file_entity_node_info_proto_rawDescData +} + +var file_entity_node_info_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_node_info_proto_goTypes = []interface{}{ + (*NodeInfo)(nil), // 0: grpc.NodeInfo +} +var file_entity_node_info_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_node_info_proto_init() } +func file_entity_node_info_proto_init() { + if File_entity_node_info_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_node_info_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*NodeInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_node_info_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_node_info_proto_goTypes, + DependencyIndexes: file_entity_node_info_proto_depIdxs, + MessageInfos: file_entity_node_info_proto_msgTypes, + }.Build() + File_entity_node_info_proto = out.File + file_entity_node_info_proto_rawDesc = nil + file_entity_node_info_proto_goTypes = nil + file_entity_node_info_proto_depIdxs = nil +} diff --git a/grpc/node_service.pb.go b/grpc/node_service.pb.go new file mode 100644 index 00000000..ebed2559 --- /dev/null +++ b/grpc/node_service.pb.go @@ -0,0 +1,104 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/node_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_node_service_proto protoreflect.FileDescriptor + +var file_services_node_service_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xfa, 0x01, + 0x0a, 0x0b, 0x4e, 0x6f, 0x64, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2b, 0x0a, + 0x08, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x30, 0x0a, 0x0d, 0x53, 0x65, + 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x0d, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x27, 0x0a, 0x04, + 0x50, 0x69, 0x6e, 0x67, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, + 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x2e, 0x0a, 0x0b, 0x55, 0x6e, + 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, + 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_node_service_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request + (*Response)(nil), // 1: grpc.Response + (*StreamMessage)(nil), // 2: grpc.StreamMessage +} +var file_services_node_service_proto_depIdxs = []int32{ + 0, // 0: grpc.NodeService.Register:input_type -> grpc.Request + 0, // 1: grpc.NodeService.SendHeartbeat:input_type -> grpc.Request + 0, // 2: grpc.NodeService.Ping:input_type -> grpc.Request + 0, // 3: grpc.NodeService.Subscribe:input_type -> grpc.Request + 0, // 4: grpc.NodeService.Unsubscribe:input_type -> grpc.Request + 1, // 5: grpc.NodeService.Register:output_type -> grpc.Response + 1, // 6: grpc.NodeService.SendHeartbeat:output_type -> grpc.Response + 1, // 7: grpc.NodeService.Ping:output_type -> grpc.Response + 2, // 8: grpc.NodeService.Subscribe:output_type -> grpc.StreamMessage + 1, // 9: grpc.NodeService.Unsubscribe:output_type -> grpc.Response + 5, // [5:10] is the sub-list for method output_type + 0, // [0:5] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_node_service_proto_init() } +func file_services_node_service_proto_init() { + if File_services_node_service_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_node_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_node_service_proto_goTypes, + DependencyIndexes: file_services_node_service_proto_depIdxs, + }.Build() + File_services_node_service_proto = out.File + file_services_node_service_proto_rawDesc = nil + file_services_node_service_proto_goTypes = nil + file_services_node_service_proto_depIdxs = nil +} diff --git a/grpc/node_service_grpc.pb.go b/grpc/node_service_grpc.pb.go new file mode 100644 index 00000000..1e0372f1 --- /dev/null +++ b/grpc/node_service_grpc.pb.go @@ -0,0 +1,277 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/node_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// NodeServiceClient is the client API for NodeService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type NodeServiceClient interface { + Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) + Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type nodeServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewNodeServiceClient(cc grpc.ClientConnInterface) NodeServiceClient { + return &nodeServiceClient{cc} +} + +func (c *nodeServiceClient) Register(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Register", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeServiceClient) SendHeartbeat(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/SendHeartbeat", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeServiceClient) Ping(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Ping", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *nodeServiceClient) Subscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (NodeService_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &NodeService_ServiceDesc.Streams[0], "/grpc.NodeService/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &nodeServiceSubscribeClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type NodeService_SubscribeClient interface { + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type nodeServiceSubscribeClient struct { + grpc.ClientStream +} + +func (x *nodeServiceSubscribeClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *nodeServiceClient) Unsubscribe(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.NodeService/Unsubscribe", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// NodeServiceServer is the server API for NodeService service. +// All implementations must embed UnimplementedNodeServiceServer +// for forward compatibility +type NodeServiceServer interface { + Register(context.Context, *Request) (*Response, error) + SendHeartbeat(context.Context, *Request) (*Response, error) + Ping(context.Context, *Request) (*Response, error) + Subscribe(*Request, NodeService_SubscribeServer) error + Unsubscribe(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedNodeServiceServer() +} + +// UnimplementedNodeServiceServer must be embedded to have forward compatible implementations. +type UnimplementedNodeServiceServer struct { +} + +func (UnimplementedNodeServiceServer) Register(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") +} +func (UnimplementedNodeServiceServer) SendHeartbeat(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method SendHeartbeat not implemented") +} +func (UnimplementedNodeServiceServer) Ping(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Ping not implemented") +} +func (UnimplementedNodeServiceServer) Subscribe(*Request, NodeService_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedNodeServiceServer) Unsubscribe(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Unsubscribe not implemented") +} +func (UnimplementedNodeServiceServer) mustEmbedUnimplementedNodeServiceServer() {} + +// UnsafeNodeServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NodeServiceServer will +// result in compilation errors. +type UnsafeNodeServiceServer interface { + mustEmbedUnimplementedNodeServiceServer() +} + +func RegisterNodeServiceServer(s grpc.ServiceRegistrar, srv NodeServiceServer) { + s.RegisterService(&NodeService_ServiceDesc, srv) +} + +func _NodeService_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).Register(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/Register", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).Register(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _NodeService_SendHeartbeat_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).SendHeartbeat(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/SendHeartbeat", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).SendHeartbeat(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _NodeService_Ping_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).Ping(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/Ping", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).Ping(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _NodeService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(Request) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(NodeServiceServer).Subscribe(m, &nodeServiceSubscribeServer{stream}) +} + +type NodeService_SubscribeServer interface { + Send(*StreamMessage) error + grpc.ServerStream +} + +type nodeServiceSubscribeServer struct { + grpc.ServerStream +} + +func (x *nodeServiceSubscribeServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func _NodeService_Unsubscribe_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(NodeServiceServer).Unsubscribe(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.NodeService/Unsubscribe", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(NodeServiceServer).Unsubscribe(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// NodeService_ServiceDesc is the grpc.ServiceDesc for NodeService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NodeService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.NodeService", + HandlerType: (*NodeServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Register", + Handler: _NodeService_Register_Handler, + }, + { + MethodName: "SendHeartbeat", + Handler: _NodeService_SendHeartbeat_Handler, + }, + { + MethodName: "Ping", + Handler: _NodeService_Ping_Handler, + }, + { + MethodName: "Unsubscribe", + Handler: _NodeService_Unsubscribe_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _NodeService_Subscribe_Handler, + ServerStreams: true, + }, + }, + Metadata: "services/node_service.proto", +} diff --git a/grpc/plugin_request.pb.go b/grpc/plugin_request.pb.go new file mode 100644 index 00000000..3db72ec1 --- /dev/null +++ b/grpc/plugin_request.pb.go @@ -0,0 +1,166 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/plugin_request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type PluginRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + NodeKey string `protobuf:"bytes,2,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *PluginRequest) Reset() { + *x = PluginRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_plugin_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PluginRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PluginRequest) ProtoMessage() {} + +func (x *PluginRequest) ProtoReflect() protoreflect.Message { + mi := &file_entity_plugin_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PluginRequest.ProtoReflect.Descriptor instead. +func (*PluginRequest) Descriptor() ([]byte, []int) { + return file_entity_plugin_request_proto_rawDescGZIP(), []int{0} +} + +func (x *PluginRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *PluginRequest) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *PluginRequest) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_entity_plugin_request_proto protoreflect.FileDescriptor + +var file_entity_plugin_request_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x5f, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x22, 0x52, 0x0a, 0x0d, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, + 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, + 0x4b, 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, + 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_plugin_request_proto_rawDescOnce sync.Once + file_entity_plugin_request_proto_rawDescData = file_entity_plugin_request_proto_rawDesc +) + +func file_entity_plugin_request_proto_rawDescGZIP() []byte { + file_entity_plugin_request_proto_rawDescOnce.Do(func() { + file_entity_plugin_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_plugin_request_proto_rawDescData) + }) + return file_entity_plugin_request_proto_rawDescData +} + +var file_entity_plugin_request_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_plugin_request_proto_goTypes = []interface{}{ + (*PluginRequest)(nil), // 0: grpc.PluginRequest +} +var file_entity_plugin_request_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_plugin_request_proto_init() } +func file_entity_plugin_request_proto_init() { + if File_entity_plugin_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_plugin_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PluginRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_plugin_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_plugin_request_proto_goTypes, + DependencyIndexes: file_entity_plugin_request_proto_depIdxs, + MessageInfos: file_entity_plugin_request_proto_msgTypes, + }.Build() + File_entity_plugin_request_proto = out.File + file_entity_plugin_request_proto_rawDesc = nil + file_entity_plugin_request_proto_goTypes = nil + file_entity_plugin_request_proto_depIdxs = nil +} diff --git a/grpc/plugin_service.pb.go b/grpc/plugin_service.pb.go new file mode 100644 index 00000000..d8974eef --- /dev/null +++ b/grpc/plugin_service.pb.go @@ -0,0 +1,96 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/plugin_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_plugin_service_proto protoreflect.FileDescriptor + +var file_services_plugin_service_proto_rawDesc = []byte{ + 0x0a, 0x1d, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x70, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x70, 0x6c, + 0x75, 0x67, 0x69, 0x6e, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xb5, 0x01, 0x0a, 0x0d, 0x50, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x31, 0x0a, 0x08, 0x52, 0x65, 0x67, 0x69, + 0x73, 0x74, 0x65, 0x72, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x50, 0x6c, 0x75, 0x67, + 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, + 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x09, 0x53, + 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, + 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, + 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x36, 0x0a, 0x04, 0x50, 0x6f, 0x6c, 0x6c, 0x12, 0x13, + 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x1a, 0x13, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x42, 0x08, + 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_plugin_service_proto_goTypes = []interface{}{ + (*PluginRequest)(nil), // 0: grpc.PluginRequest + (*StreamMessage)(nil), // 1: grpc.StreamMessage + (*Response)(nil), // 2: grpc.Response +} +var file_services_plugin_service_proto_depIdxs = []int32{ + 0, // 0: grpc.PluginService.Register:input_type -> grpc.PluginRequest + 0, // 1: grpc.PluginService.Subscribe:input_type -> grpc.PluginRequest + 1, // 2: grpc.PluginService.Poll:input_type -> grpc.StreamMessage + 2, // 3: grpc.PluginService.Register:output_type -> grpc.Response + 1, // 4: grpc.PluginService.Subscribe:output_type -> grpc.StreamMessage + 1, // 5: grpc.PluginService.Poll:output_type -> grpc.StreamMessage + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_plugin_service_proto_init() } +func file_services_plugin_service_proto_init() { + if File_services_plugin_service_proto != nil { + return + } + file_entity_plugin_request_proto_init() + file_entity_response_proto_init() + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_plugin_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_plugin_service_proto_goTypes, + DependencyIndexes: file_services_plugin_service_proto_depIdxs, + }.Build() + File_services_plugin_service_proto = out.File + file_services_plugin_service_proto_rawDesc = nil + file_services_plugin_service_proto_goTypes = nil + file_services_plugin_service_proto_depIdxs = nil +} diff --git a/grpc/plugin_service_grpc.pb.go b/grpc/plugin_service_grpc.pb.go new file mode 100644 index 00000000..06b93262 --- /dev/null +++ b/grpc/plugin_service_grpc.pb.go @@ -0,0 +1,237 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/plugin_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// PluginServiceClient is the client API for PluginService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type PluginServiceClient interface { + Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) + Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) + Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) +} + +type pluginServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewPluginServiceClient(cc grpc.ClientConnInterface) PluginServiceClient { + return &pluginServiceClient{cc} +} + +func (c *pluginServiceClient) Register(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.PluginService/Register", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *pluginServiceClient) Subscribe(ctx context.Context, in *PluginRequest, opts ...grpc.CallOption) (PluginService_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[0], "/grpc.PluginService/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &pluginServiceSubscribeClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type PluginService_SubscribeClient interface { + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type pluginServiceSubscribeClient struct { + grpc.ClientStream +} + +func (x *pluginServiceSubscribeClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *pluginServiceClient) Poll(ctx context.Context, opts ...grpc.CallOption) (PluginService_PollClient, error) { + stream, err := c.cc.NewStream(ctx, &PluginService_ServiceDesc.Streams[1], "/grpc.PluginService/Poll", opts...) + if err != nil { + return nil, err + } + x := &pluginServicePollClient{stream} + return x, nil +} + +type PluginService_PollClient interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ClientStream +} + +type pluginServicePollClient struct { + grpc.ClientStream +} + +func (x *pluginServicePollClient) Send(m *StreamMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *pluginServicePollClient) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// PluginServiceServer is the server API for PluginService service. +// All implementations must embed UnimplementedPluginServiceServer +// for forward compatibility +type PluginServiceServer interface { + Register(context.Context, *PluginRequest) (*Response, error) + Subscribe(*PluginRequest, PluginService_SubscribeServer) error + Poll(PluginService_PollServer) error + mustEmbedUnimplementedPluginServiceServer() +} + +// UnimplementedPluginServiceServer must be embedded to have forward compatible implementations. +type UnimplementedPluginServiceServer struct { +} + +func (UnimplementedPluginServiceServer) Register(context.Context, *PluginRequest) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Register not implemented") +} +func (UnimplementedPluginServiceServer) Subscribe(*PluginRequest, PluginService_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedPluginServiceServer) Poll(PluginService_PollServer) error { + return status.Errorf(codes.Unimplemented, "method Poll not implemented") +} +func (UnimplementedPluginServiceServer) mustEmbedUnimplementedPluginServiceServer() {} + +// UnsafePluginServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to PluginServiceServer will +// result in compilation errors. +type UnsafePluginServiceServer interface { + mustEmbedUnimplementedPluginServiceServer() +} + +func RegisterPluginServiceServer(s grpc.ServiceRegistrar, srv PluginServiceServer) { + s.RegisterService(&PluginService_ServiceDesc, srv) +} + +func _PluginService_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PluginRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(PluginServiceServer).Register(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.PluginService/Register", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(PluginServiceServer).Register(ctx, req.(*PluginRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _PluginService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(PluginRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(PluginServiceServer).Subscribe(m, &pluginServiceSubscribeServer{stream}) +} + +type PluginService_SubscribeServer interface { + Send(*StreamMessage) error + grpc.ServerStream +} + +type pluginServiceSubscribeServer struct { + grpc.ServerStream +} + +func (x *pluginServiceSubscribeServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func _PluginService_Poll_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(PluginServiceServer).Poll(&pluginServicePollServer{stream}) +} + +type PluginService_PollServer interface { + Send(*StreamMessage) error + Recv() (*StreamMessage, error) + grpc.ServerStream +} + +type pluginServicePollServer struct { + grpc.ServerStream +} + +func (x *pluginServicePollServer) Send(m *StreamMessage) error { + return x.ServerStream.SendMsg(m) +} + +func (x *pluginServicePollServer) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// PluginService_ServiceDesc is the grpc.ServiceDesc for PluginService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var PluginService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.PluginService", + HandlerType: (*PluginServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Register", + Handler: _PluginService_Register_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _PluginService_Subscribe_Handler, + ServerStreams: true, + }, + { + StreamName: "Poll", + Handler: _PluginService_Poll_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "services/plugin_service.proto", +} diff --git a/grpc/proto/entity/dependencies_service_v2_request.proto b/grpc/proto/entity/dependencies_service_v2_request.proto new file mode 100644 index 00000000..c7cfa79d --- /dev/null +++ b/grpc/proto/entity/dependencies_service_v2_request.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Dependency { + string name = 1; + string version = 2; +} + +message DependenciesServiceV2ConnectRequest { + string node_key = 1; +} + +message DependenciesServiceV2SyncRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; +} + +message DependenciesServiceV2InstallRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; + string proxy = 4; +} + +message DependenciesServiceV2UninstallRequest { + string node_key = 1; + string lang = 2; + repeated Dependency dependencies = 3; +} \ No newline at end of file diff --git a/grpc/proto/entity/model_service_v2_request.proto b/grpc/proto/entity/model_service_v2_request.proto new file mode 100644 index 00000000..f2933ca7 --- /dev/null +++ b/grpc/proto/entity/model_service_v2_request.proto @@ -0,0 +1,95 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message ModelServiceV2GetByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; +} + +message ModelServiceV2GetOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes find_options = 4; +} + +message ModelServiceV2GetManyRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes find_options = 4; +} + +message ModelServiceV2DeleteByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; +} + +message ModelServiceV2DeleteOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; +} + +message ModelServiceV2DeleteManyRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; +} + +message ModelServiceV2UpdateByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; + bytes update = 4; +} + +message ModelServiceV2UpdateOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes update = 4; +} + +message ModelServiceV2UpdateManyRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes update = 4; +} + +message ModelServiceV2ReplaceByIdRequest { + string node_key = 1; + string model_type = 2; + string id = 3; + bytes model = 4; +} + +message ModelServiceV2ReplaceOneRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; + bytes model = 4; +} + +message ModelServiceV2InsertOneRequest { + string node_key = 1; + string model_type = 2; + bytes model = 3; +} + +message ModelServiceV2InsertManyRequest { + string node_key = 1; + string model_type = 2; + bytes models = 3; +} + +message ModelServiceV2CountRequest { + string node_key = 1; + string model_type = 2; + bytes query = 3; +} diff --git a/grpc/proto/entity/node_info.proto b/grpc/proto/entity/node_info.proto new file mode 100644 index 00000000..e289d29d --- /dev/null +++ b/grpc/proto/entity/node_info.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message NodeInfo { + string key = 1; + bool is_master = 2; +} diff --git a/grpc/proto/entity/plugin_request.proto b/grpc/proto/entity/plugin_request.proto new file mode 100644 index 00000000..9359028e --- /dev/null +++ b/grpc/proto/entity/plugin_request.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message PluginRequest { + string name = 1; + string node_key = 2; + bytes data = 3; +} diff --git a/grpc/proto/entity/request.proto b/grpc/proto/entity/request.proto new file mode 100644 index 00000000..78b88e7d --- /dev/null +++ b/grpc/proto/entity/request.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Request { + string node_key = 1; + bytes data = 2; +} diff --git a/grpc/proto/entity/response.proto b/grpc/proto/entity/response.proto new file mode 100644 index 00000000..9145a1b0 --- /dev/null +++ b/grpc/proto/entity/response.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/response_code.proto"; + +package grpc; +option go_package = ".;grpc"; + +message Response { + ResponseCode code = 1; + string message = 2; + bytes data = 3; + string error = 4; + int64 total = 5; +} diff --git a/grpc/proto/entity/response_code.proto b/grpc/proto/entity/response_code.proto new file mode 100644 index 00000000..cb7398fc --- /dev/null +++ b/grpc/proto/entity/response_code.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +option go_package = ".;grpc"; + +enum ResponseCode { + OK = 0; + ERROR = 1; +} \ No newline at end of file diff --git a/grpc/proto/entity/stream_message.proto b/grpc/proto/entity/stream_message.proto new file mode 100644 index 00000000..44cd7af4 --- /dev/null +++ b/grpc/proto/entity/stream_message.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "entity/stream_message_code.proto"; + +package grpc; +option go_package = ".;grpc"; + +message StreamMessage { + StreamMessageCode code = 1; + string node_key = 2; + string key = 3; + string from = 4; + string to = 5; + bytes data = 6; + string error = 7; +} diff --git a/grpc/proto/entity/stream_message_code.proto b/grpc/proto/entity/stream_message_code.proto new file mode 100644 index 00000000..f2c41882 --- /dev/null +++ b/grpc/proto/entity/stream_message_code.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +enum StreamMessageCode { + // ping worker nodes to check their health + PING = 0; + // ask worker node(s) to run task with given id + RUN_TASK = 1; + // ask worker node(s) to cancel task with given id + CANCEL_TASK = 2; + // insert data + INSERT_DATA = 3; + // insert logs + INSERT_LOGS = 4; + // send event + SEND_EVENT = 5; + // install plugin + INSTALL_PLUGIN = 6; + // uninstall plugin + UNINSTALL_PLUGIN = 7; + // start plugin + START_PLUGIN = 8; + // stop plugin + STOP_PLUGIN = 9; + // connect + CONNECT = 10; + // disconnect + DISCONNECT = 11; + // send + SEND = 12; +} diff --git a/grpc/proto/entity/stream_message_data_task.proto b/grpc/proto/entity/stream_message_data_task.proto new file mode 100644 index 00000000..b6c082d9 --- /dev/null +++ b/grpc/proto/entity/stream_message_data_task.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message StreamMessageDataTask { + string task_id = 1; + string data = 2; +} diff --git a/grpc/proto/models/node.proto b/grpc/proto/models/node.proto new file mode 100644 index 00000000..aa5312cc --- /dev/null +++ b/grpc/proto/models/node.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Node { + string _id = 1; + string name = 2; + string ip = 3; + string port = 5; + string mac = 6; + string hostname = 7; + string description = 8; + string key = 9; + bool is_master = 11; + string update_ts = 12; + string create_ts = 13; + int64 update_ts_unix = 14; +} diff --git a/grpc/proto/models/task.proto b/grpc/proto/models/task.proto new file mode 100644 index 00000000..52ea7056 --- /dev/null +++ b/grpc/proto/models/task.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package grpc; +option go_package = ".;grpc"; + +message Task { + string _id = 1; + string spider_id = 2; + string status = 5; + string node_id = 6; + string cmd = 8; + string param = 9; + string error = 10; + int32 pid = 16; + string run_type = 17; + string schedule_id = 18; + string type = 19; +} \ No newline at end of file diff --git a/grpc/proto/services/dependency_service_v2.proto b/grpc/proto/services/dependency_service_v2.proto new file mode 100644 index 00000000..dea310d6 --- /dev/null +++ b/grpc/proto/services/dependency_service_v2.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/dependencies_service_v2_request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service DependencyServiceV2 { + rpc Connect(stream DependenciesServiceV2ConnectRequest) returns (Response){}; + rpc Sync(DependenciesServiceV2SyncRequest) returns (Response){}; + rpc Install(stream DependenciesServiceV2InstallRequest) returns (stream Response){}; + rpc UninstallDependencies(stream DependenciesServiceV2UninstallRequest) returns (stream Response){}; +} diff --git a/grpc/proto/services/message_service.proto b/grpc/proto/services/message_service.proto new file mode 100644 index 00000000..41207403 --- /dev/null +++ b/grpc/proto/services/message_service.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service MessageService { + rpc Connect(stream StreamMessage) returns (stream StreamMessage){}; +} diff --git a/grpc/proto/services/model_base_service.proto b/grpc/proto/services/model_base_service.proto new file mode 100644 index 00000000..0836659a --- /dev/null +++ b/grpc/proto/services/model_base_service.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service ModelBaseService { + rpc GetById(Request) returns (Response){}; + rpc Get(Request) returns (Response){}; + rpc GetList(Request) returns (Response){}; + rpc DeleteById(Request) returns (Response){}; + rpc Delete(Request) returns (Response){}; + rpc DeleteList(Request) returns (Response){}; + rpc ForceDeleteList(Request) returns (Response){}; + rpc UpdateById(Request) returns (Response){}; + rpc Update(Request) returns (Response){}; + rpc UpdateDoc(Request) returns (Response){}; + rpc Insert(Request) returns (Response){}; + rpc Count(Request) returns (Response){}; +} diff --git a/grpc/proto/services/model_base_service_v2.proto b/grpc/proto/services/model_base_service_v2.proto new file mode 100644 index 00000000..e0b309f7 --- /dev/null +++ b/grpc/proto/services/model_base_service_v2.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +import "entity/model_service_v2_request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service ModelBaseServiceV2 { + rpc GetById(ModelServiceV2GetByIdRequest) returns (Response){}; + rpc GetOne(ModelServiceV2GetOneRequest) returns (Response){}; + rpc GetMany(ModelServiceV2GetManyRequest) returns (Response){}; + rpc DeleteById(ModelServiceV2DeleteByIdRequest) returns (Response){}; + rpc DeleteOne(ModelServiceV2DeleteOneRequest) returns (Response){}; + rpc DeleteMany(ModelServiceV2DeleteManyRequest) returns (Response){}; + rpc UpdateById(ModelServiceV2UpdateByIdRequest) returns (Response){}; + rpc UpdateOne(ModelServiceV2UpdateOneRequest) returns (Response){}; + rpc UpdateMany(ModelServiceV2UpdateManyRequest) returns (Response){}; + rpc ReplaceById(ModelServiceV2ReplaceByIdRequest) returns (Response){}; + rpc ReplaceOne(ModelServiceV2ReplaceOneRequest) returns (Response){}; + rpc InsertOne(ModelServiceV2InsertOneRequest) returns (Response){}; + rpc InsertMany(ModelServiceV2InsertManyRequest) returns (Response){}; + rpc Count(ModelServiceV2CountRequest) returns (Response){}; +} diff --git a/grpc/proto/services/model_delegate.proto b/grpc/proto/services/model_delegate.proto new file mode 100644 index 00000000..1b60477c --- /dev/null +++ b/grpc/proto/services/model_delegate.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; + +package grpc; +option go_package = ".;grpc"; + +service ModelDelegate { + rpc Do(Request) returns (Response){}; +} diff --git a/grpc/proto/services/node_service.proto b/grpc/proto/services/node_service.proto new file mode 100644 index 00000000..4b43d083 --- /dev/null +++ b/grpc/proto/services/node_service.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service NodeService { + rpc Register(Request) returns (Response){}; + rpc SendHeartbeat(Request) returns (Response){}; + rpc Ping(Request) returns (Response){}; + rpc Subscribe(Request) returns (stream StreamMessage){}; + rpc Unsubscribe(Request) returns (Response){}; +} diff --git a/grpc/proto/services/plugin_service.proto b/grpc/proto/services/plugin_service.proto new file mode 100644 index 00000000..5e41977e --- /dev/null +++ b/grpc/proto/services/plugin_service.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/plugin_request.proto"; +import "entity/response.proto"; +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service PluginService { + rpc Register(PluginRequest) returns (Response){}; + rpc Subscribe(PluginRequest) returns (stream StreamMessage){}; + rpc Poll(stream StreamMessage) returns (stream StreamMessage){}; +} diff --git a/grpc/proto/services/task_service.proto b/grpc/proto/services/task_service.proto new file mode 100644 index 00000000..f004e565 --- /dev/null +++ b/grpc/proto/services/task_service.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +import "entity/request.proto"; +import "entity/response.proto"; +import "entity/stream_message.proto"; + +package grpc; +option go_package = ".;grpc"; + +service TaskService { + rpc Subscribe(stream StreamMessage) returns (Response){}; + rpc Fetch(Request) returns (Response){}; + rpc SendNotification(Request) returns (Response){}; +} diff --git a/grpc/request.pb.go b/grpc/request.pb.go new file mode 100644 index 00000000..3b541554 --- /dev/null +++ b/grpc/request.pb.go @@ -0,0 +1,156 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/request.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Request struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + NodeKey string `protobuf:"bytes,1,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Data []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *Request) Reset() { + *x = Request{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_request_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Request) ProtoMessage() {} + +func (x *Request) ProtoReflect() protoreflect.Message { + mi := &file_entity_request_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Request.ProtoReflect.Descriptor instead. +func (*Request) Descriptor() ([]byte, []int) { + return file_entity_request_proto_rawDescGZIP(), []int{0} +} + +func (x *Request) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *Request) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +var File_entity_request_proto protoreflect.FileDescriptor + +var file_entity_request_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x38, 0x0a, 0x07, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, + 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, + 0x65, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_request_proto_rawDescOnce sync.Once + file_entity_request_proto_rawDescData = file_entity_request_proto_rawDesc +) + +func file_entity_request_proto_rawDescGZIP() []byte { + file_entity_request_proto_rawDescOnce.Do(func() { + file_entity_request_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_request_proto_rawDescData) + }) + return file_entity_request_proto_rawDescData +} + +var file_entity_request_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_request_proto_goTypes = []interface{}{ + (*Request)(nil), // 0: grpc.Request +} +var file_entity_request_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_request_proto_init() } +func file_entity_request_proto_init() { + if File_entity_request_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_request_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Request); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_request_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_request_proto_goTypes, + DependencyIndexes: file_entity_request_proto_depIdxs, + MessageInfos: file_entity_request_proto_msgTypes, + }.Build() + File_entity_request_proto = out.File + file_entity_request_proto_rawDesc = nil + file_entity_request_proto_goTypes = nil + file_entity_request_proto_depIdxs = nil +} diff --git a/grpc/response.pb.go b/grpc/response.pb.go new file mode 100644 index 00000000..ad32f869 --- /dev/null +++ b/grpc/response.pb.go @@ -0,0 +1,190 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/response.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Response struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code ResponseCode `protobuf:"varint,1,opt,name=code,proto3,enum=ResponseCode" json:"code,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` + Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` + Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"` + Total int64 `protobuf:"varint,5,opt,name=total,proto3" json:"total,omitempty"` +} + +func (x *Response) Reset() { + *x = Response{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_response_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Response) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Response) ProtoMessage() {} + +func (x *Response) ProtoReflect() protoreflect.Message { + mi := &file_entity_response_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Response.ProtoReflect.Descriptor instead. +func (*Response) Descriptor() ([]byte, []int) { + return file_entity_response_proto_rawDescGZIP(), []int{0} +} + +func (x *Response) GetCode() ResponseCode { + if x != nil { + return x.Code + } + return ResponseCode_OK +} + +func (x *Response) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *Response) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *Response) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Response) GetTotal() int64 { + if x != nil { + return x.Total + } + return 0 +} + +var File_entity_response_proto protoreflect.FileDescriptor + +var file_entity_response_proto_rawDesc = []byte{ + 0x0a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x1a, 0x1a, 0x65, + 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x5f, 0x63, + 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x87, 0x01, 0x0a, 0x08, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0d, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, + 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x14, 0x0a, + 0x05, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x74, 0x6f, + 0x74, 0x61, 0x6c, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_response_proto_rawDescOnce sync.Once + file_entity_response_proto_rawDescData = file_entity_response_proto_rawDesc +) + +func file_entity_response_proto_rawDescGZIP() []byte { + file_entity_response_proto_rawDescOnce.Do(func() { + file_entity_response_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_response_proto_rawDescData) + }) + return file_entity_response_proto_rawDescData +} + +var file_entity_response_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_response_proto_goTypes = []interface{}{ + (*Response)(nil), // 0: grpc.Response + (ResponseCode)(0), // 1: ResponseCode +} +var file_entity_response_proto_depIdxs = []int32{ + 1, // 0: grpc.Response.code:type_name -> ResponseCode + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_entity_response_proto_init() } +func file_entity_response_proto_init() { + if File_entity_response_proto != nil { + return + } + file_entity_response_code_proto_init() + if !protoimpl.UnsafeEnabled { + file_entity_response_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Response); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_response_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_response_proto_goTypes, + DependencyIndexes: file_entity_response_proto_depIdxs, + MessageInfos: file_entity_response_proto_msgTypes, + }.Build() + File_entity_response_proto = out.File + file_entity_response_proto_rawDesc = nil + file_entity_response_proto_goTypes = nil + file_entity_response_proto_depIdxs = nil +} diff --git a/grpc/response_code.pb.go b/grpc/response_code.pb.go new file mode 100644 index 00000000..9c9700a8 --- /dev/null +++ b/grpc/response_code.pb.go @@ -0,0 +1,132 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/response_code.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type ResponseCode int32 + +const ( + ResponseCode_OK ResponseCode = 0 + ResponseCode_ERROR ResponseCode = 1 +) + +// Enum value maps for ResponseCode. +var ( + ResponseCode_name = map[int32]string{ + 0: "OK", + 1: "ERROR", + } + ResponseCode_value = map[string]int32{ + "OK": 0, + "ERROR": 1, + } +) + +func (x ResponseCode) Enum() *ResponseCode { + p := new(ResponseCode) + *p = x + return p +} + +func (x ResponseCode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ResponseCode) Descriptor() protoreflect.EnumDescriptor { + return file_entity_response_code_proto_enumTypes[0].Descriptor() +} + +func (ResponseCode) Type() protoreflect.EnumType { + return &file_entity_response_code_proto_enumTypes[0] +} + +func (x ResponseCode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ResponseCode.Descriptor instead. +func (ResponseCode) EnumDescriptor() ([]byte, []int) { + return file_entity_response_code_proto_rawDescGZIP(), []int{0} +} + +var File_entity_response_code_proto protoreflect.FileDescriptor + +var file_entity_response_code_proto_rawDesc = []byte{ + 0x0a, 0x1a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2a, 0x21, 0x0a, 0x0c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x06, 0x0a, 0x02, + 0x4f, 0x4b, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x01, 0x42, + 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_entity_response_code_proto_rawDescOnce sync.Once + file_entity_response_code_proto_rawDescData = file_entity_response_code_proto_rawDesc +) + +func file_entity_response_code_proto_rawDescGZIP() []byte { + file_entity_response_code_proto_rawDescOnce.Do(func() { + file_entity_response_code_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_response_code_proto_rawDescData) + }) + return file_entity_response_code_proto_rawDescData +} + +var file_entity_response_code_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_entity_response_code_proto_goTypes = []interface{}{ + (ResponseCode)(0), // 0: ResponseCode +} +var file_entity_response_code_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_response_code_proto_init() } +func file_entity_response_code_proto_init() { + if File_entity_response_code_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_response_code_proto_rawDesc, + NumEnums: 1, + NumMessages: 0, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_response_code_proto_goTypes, + DependencyIndexes: file_entity_response_code_proto_depIdxs, + EnumInfos: file_entity_response_code_proto_enumTypes, + }.Build() + File_entity_response_code_proto = out.File + file_entity_response_code_proto_rawDesc = nil + file_entity_response_code_proto_goTypes = nil + file_entity_response_code_proto_depIdxs = nil +} diff --git a/grpc/stream_message.pb.go b/grpc/stream_message.pb.go new file mode 100644 index 00000000..2e563dfc --- /dev/null +++ b/grpc/stream_message.pb.go @@ -0,0 +1,210 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/stream_message.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type StreamMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Code StreamMessageCode `protobuf:"varint,1,opt,name=code,proto3,enum=grpc.StreamMessageCode" json:"code,omitempty"` + NodeKey string `protobuf:"bytes,2,opt,name=node_key,json=nodeKey,proto3" json:"node_key,omitempty"` + Key string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"` + From string `protobuf:"bytes,4,opt,name=from,proto3" json:"from,omitempty"` + To string `protobuf:"bytes,5,opt,name=to,proto3" json:"to,omitempty"` + Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` + Error string `protobuf:"bytes,7,opt,name=error,proto3" json:"error,omitempty"` +} + +func (x *StreamMessage) Reset() { + *x = StreamMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_stream_message_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamMessage) ProtoMessage() {} + +func (x *StreamMessage) ProtoReflect() protoreflect.Message { + mi := &file_entity_stream_message_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamMessage.ProtoReflect.Descriptor instead. +func (*StreamMessage) Descriptor() ([]byte, []int) { + return file_entity_stream_message_proto_rawDescGZIP(), []int{0} +} + +func (x *StreamMessage) GetCode() StreamMessageCode { + if x != nil { + return x.Code + } + return StreamMessageCode_PING +} + +func (x *StreamMessage) GetNodeKey() string { + if x != nil { + return x.NodeKey + } + return "" +} + +func (x *StreamMessage) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *StreamMessage) GetFrom() string { + if x != nil { + return x.From + } + return "" +} + +func (x *StreamMessage) GetTo() string { + if x != nil { + return x.To + } + return "" +} + +func (x *StreamMessage) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *StreamMessage) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +var File_entity_stream_message_proto protoreflect.FileDescriptor + +var file_entity_stream_message_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x1a, 0x20, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, + 0x61, 0x6d, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb7, 0x01, 0x0a, 0x0d, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x2b, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x17, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, + 0x63, 0x6f, 0x64, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x12, 0x0a, 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x66, 0x72, 0x6f, 0x6d, 0x12, 0x0e, 0x0a, 0x02, 0x74, 0x6f, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x02, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, + 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_entity_stream_message_proto_rawDescOnce sync.Once + file_entity_stream_message_proto_rawDescData = file_entity_stream_message_proto_rawDesc +) + +func file_entity_stream_message_proto_rawDescGZIP() []byte { + file_entity_stream_message_proto_rawDescOnce.Do(func() { + file_entity_stream_message_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_stream_message_proto_rawDescData) + }) + return file_entity_stream_message_proto_rawDescData +} + +var file_entity_stream_message_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_stream_message_proto_goTypes = []interface{}{ + (*StreamMessage)(nil), // 0: grpc.StreamMessage + (StreamMessageCode)(0), // 1: grpc.StreamMessageCode +} +var file_entity_stream_message_proto_depIdxs = []int32{ + 1, // 0: grpc.StreamMessage.code:type_name -> grpc.StreamMessageCode + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_entity_stream_message_proto_init() } +func file_entity_stream_message_proto_init() { + if File_entity_stream_message_proto != nil { + return + } + file_entity_stream_message_code_proto_init() + if !protoimpl.UnsafeEnabled { + file_entity_stream_message_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_stream_message_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_stream_message_proto_goTypes, + DependencyIndexes: file_entity_stream_message_proto_depIdxs, + MessageInfos: file_entity_stream_message_proto_msgTypes, + }.Build() + File_entity_stream_message_proto = out.File + file_entity_stream_message_proto_rawDesc = nil + file_entity_stream_message_proto_goTypes = nil + file_entity_stream_message_proto_depIdxs = nil +} diff --git a/grpc/stream_message_code.pb.go b/grpc/stream_message_code.pb.go new file mode 100644 index 00000000..90e1f9f6 --- /dev/null +++ b/grpc/stream_message_code.pb.go @@ -0,0 +1,190 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/stream_message_code.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type StreamMessageCode int32 + +const ( + // ping worker nodes to check their health + StreamMessageCode_PING StreamMessageCode = 0 + // ask worker node(s) to run task with given id + StreamMessageCode_RUN_TASK StreamMessageCode = 1 + // ask worker node(s) to cancel task with given id + StreamMessageCode_CANCEL_TASK StreamMessageCode = 2 + // insert data + StreamMessageCode_INSERT_DATA StreamMessageCode = 3 + // insert logs + StreamMessageCode_INSERT_LOGS StreamMessageCode = 4 + // send event + StreamMessageCode_SEND_EVENT StreamMessageCode = 5 + // install plugin + StreamMessageCode_INSTALL_PLUGIN StreamMessageCode = 6 + // uninstall plugin + StreamMessageCode_UNINSTALL_PLUGIN StreamMessageCode = 7 + // start plugin + StreamMessageCode_START_PLUGIN StreamMessageCode = 8 + // stop plugin + StreamMessageCode_STOP_PLUGIN StreamMessageCode = 9 + // connect + StreamMessageCode_CONNECT StreamMessageCode = 10 + // disconnect + StreamMessageCode_DISCONNECT StreamMessageCode = 11 + // send + StreamMessageCode_SEND StreamMessageCode = 12 +) + +// Enum value maps for StreamMessageCode. +var ( + StreamMessageCode_name = map[int32]string{ + 0: "PING", + 1: "RUN_TASK", + 2: "CANCEL_TASK", + 3: "INSERT_DATA", + 4: "INSERT_LOGS", + 5: "SEND_EVENT", + 6: "INSTALL_PLUGIN", + 7: "UNINSTALL_PLUGIN", + 8: "START_PLUGIN", + 9: "STOP_PLUGIN", + 10: "CONNECT", + 11: "DISCONNECT", + 12: "SEND", + } + StreamMessageCode_value = map[string]int32{ + "PING": 0, + "RUN_TASK": 1, + "CANCEL_TASK": 2, + "INSERT_DATA": 3, + "INSERT_LOGS": 4, + "SEND_EVENT": 5, + "INSTALL_PLUGIN": 6, + "UNINSTALL_PLUGIN": 7, + "START_PLUGIN": 8, + "STOP_PLUGIN": 9, + "CONNECT": 10, + "DISCONNECT": 11, + "SEND": 12, + } +) + +func (x StreamMessageCode) Enum() *StreamMessageCode { + p := new(StreamMessageCode) + *p = x + return p +} + +func (x StreamMessageCode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (StreamMessageCode) Descriptor() protoreflect.EnumDescriptor { + return file_entity_stream_message_code_proto_enumTypes[0].Descriptor() +} + +func (StreamMessageCode) Type() protoreflect.EnumType { + return &file_entity_stream_message_code_proto_enumTypes[0] +} + +func (x StreamMessageCode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use StreamMessageCode.Descriptor instead. +func (StreamMessageCode) EnumDescriptor() ([]byte, []int) { + return file_entity_stream_message_code_proto_rawDescGZIP(), []int{0} +} + +var File_entity_stream_message_code_proto protoreflect.FileDescriptor + +var file_entity_stream_message_code_proto_rawDesc = []byte{ + 0x0a, 0x20, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x2a, 0xe2, 0x01, 0x0a, 0x11, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x08, + 0x0a, 0x04, 0x50, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x52, 0x55, 0x4e, 0x5f, + 0x54, 0x41, 0x53, 0x4b, 0x10, 0x01, 0x12, 0x0f, 0x0a, 0x0b, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, + 0x5f, 0x54, 0x41, 0x53, 0x4b, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x53, 0x45, 0x52, + 0x54, 0x5f, 0x44, 0x41, 0x54, 0x41, 0x10, 0x03, 0x12, 0x0f, 0x0a, 0x0b, 0x49, 0x4e, 0x53, 0x45, + 0x52, 0x54, 0x5f, 0x4c, 0x4f, 0x47, 0x53, 0x10, 0x04, 0x12, 0x0e, 0x0a, 0x0a, 0x53, 0x45, 0x4e, + 0x44, 0x5f, 0x45, 0x56, 0x45, 0x4e, 0x54, 0x10, 0x05, 0x12, 0x12, 0x0a, 0x0e, 0x49, 0x4e, 0x53, + 0x54, 0x41, 0x4c, 0x4c, 0x5f, 0x50, 0x4c, 0x55, 0x47, 0x49, 0x4e, 0x10, 0x06, 0x12, 0x14, 0x0a, + 0x10, 0x55, 0x4e, 0x49, 0x4e, 0x53, 0x54, 0x41, 0x4c, 0x4c, 0x5f, 0x50, 0x4c, 0x55, 0x47, 0x49, + 0x4e, 0x10, 0x07, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x50, 0x4c, 0x55, + 0x47, 0x49, 0x4e, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x53, 0x54, 0x4f, 0x50, 0x5f, 0x50, 0x4c, + 0x55, 0x47, 0x49, 0x4e, 0x10, 0x09, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x10, 0x0a, 0x12, 0x0e, 0x0a, 0x0a, 0x44, 0x49, 0x53, 0x43, 0x4f, 0x4e, 0x4e, 0x45, 0x43, + 0x54, 0x10, 0x0b, 0x12, 0x08, 0x0a, 0x04, 0x53, 0x45, 0x4e, 0x44, 0x10, 0x0c, 0x42, 0x08, 0x5a, + 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_stream_message_code_proto_rawDescOnce sync.Once + file_entity_stream_message_code_proto_rawDescData = file_entity_stream_message_code_proto_rawDesc +) + +func file_entity_stream_message_code_proto_rawDescGZIP() []byte { + file_entity_stream_message_code_proto_rawDescOnce.Do(func() { + file_entity_stream_message_code_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_stream_message_code_proto_rawDescData) + }) + return file_entity_stream_message_code_proto_rawDescData +} + +var file_entity_stream_message_code_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_entity_stream_message_code_proto_goTypes = []interface{}{ + (StreamMessageCode)(0), // 0: grpc.StreamMessageCode +} +var file_entity_stream_message_code_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_stream_message_code_proto_init() } +func file_entity_stream_message_code_proto_init() { + if File_entity_stream_message_code_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_stream_message_code_proto_rawDesc, + NumEnums: 1, + NumMessages: 0, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_stream_message_code_proto_goTypes, + DependencyIndexes: file_entity_stream_message_code_proto_depIdxs, + EnumInfos: file_entity_stream_message_code_proto_enumTypes, + }.Build() + File_entity_stream_message_code_proto = out.File + file_entity_stream_message_code_proto_rawDesc = nil + file_entity_stream_message_code_proto_goTypes = nil + file_entity_stream_message_code_proto_depIdxs = nil +} diff --git a/grpc/stream_message_data_task.pb.go b/grpc/stream_message_data_task.pb.go new file mode 100644 index 00000000..e2ffac6e --- /dev/null +++ b/grpc/stream_message_data_task.pb.go @@ -0,0 +1,158 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: entity/stream_message_data_task.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type StreamMessageDataTask struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TaskId string `protobuf:"bytes,1,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + Data string `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` +} + +func (x *StreamMessageDataTask) Reset() { + *x = StreamMessageDataTask{} + if protoimpl.UnsafeEnabled { + mi := &file_entity_stream_message_data_task_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *StreamMessageDataTask) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamMessageDataTask) ProtoMessage() {} + +func (x *StreamMessageDataTask) ProtoReflect() protoreflect.Message { + mi := &file_entity_stream_message_data_task_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamMessageDataTask.ProtoReflect.Descriptor instead. +func (*StreamMessageDataTask) Descriptor() ([]byte, []int) { + return file_entity_stream_message_data_task_proto_rawDescGZIP(), []int{0} +} + +func (x *StreamMessageDataTask) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + +func (x *StreamMessageDataTask) GetData() string { + if x != nil { + return x.Data + } + return "" +} + +var File_entity_stream_message_data_task_proto protoreflect.FileDescriptor + +var file_entity_stream_message_data_task_proto_rawDesc = []byte{ + 0x0a, 0x25, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x74, 0x61, 0x73, + 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x44, 0x0a, + 0x15, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x44, 0x61, + 0x74, 0x61, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, + 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x64, + 0x61, 0x74, 0x61, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_entity_stream_message_data_task_proto_rawDescOnce sync.Once + file_entity_stream_message_data_task_proto_rawDescData = file_entity_stream_message_data_task_proto_rawDesc +) + +func file_entity_stream_message_data_task_proto_rawDescGZIP() []byte { + file_entity_stream_message_data_task_proto_rawDescOnce.Do(func() { + file_entity_stream_message_data_task_proto_rawDescData = protoimpl.X.CompressGZIP(file_entity_stream_message_data_task_proto_rawDescData) + }) + return file_entity_stream_message_data_task_proto_rawDescData +} + +var file_entity_stream_message_data_task_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_entity_stream_message_data_task_proto_goTypes = []interface{}{ + (*StreamMessageDataTask)(nil), // 0: grpc.StreamMessageDataTask +} +var file_entity_stream_message_data_task_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_entity_stream_message_data_task_proto_init() } +func file_entity_stream_message_data_task_proto_init() { + if File_entity_stream_message_data_task_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_entity_stream_message_data_task_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*StreamMessageDataTask); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_entity_stream_message_data_task_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_entity_stream_message_data_task_proto_goTypes, + DependencyIndexes: file_entity_stream_message_data_task_proto_depIdxs, + MessageInfos: file_entity_stream_message_data_task_proto_msgTypes, + }.Build() + File_entity_stream_message_data_task_proto = out.File + file_entity_stream_message_data_task_proto_rawDesc = nil + file_entity_stream_message_data_task_proto_goTypes = nil + file_entity_stream_message_data_task_proto_depIdxs = nil +} diff --git a/grpc/task.pb.go b/grpc/task.pb.go new file mode 100644 index 00000000..af5aa474 --- /dev/null +++ b/grpc/task.pb.go @@ -0,0 +1,241 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: models/task.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +type Task struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + XId string `protobuf:"bytes,1,opt,name=_id,json=Id,proto3" json:"_id,omitempty"` + SpiderId string `protobuf:"bytes,2,opt,name=spider_id,json=spiderId,proto3" json:"spider_id,omitempty"` + Status string `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` + NodeId string `protobuf:"bytes,6,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + Cmd string `protobuf:"bytes,8,opt,name=cmd,proto3" json:"cmd,omitempty"` + Param string `protobuf:"bytes,9,opt,name=param,proto3" json:"param,omitempty"` + Error string `protobuf:"bytes,10,opt,name=error,proto3" json:"error,omitempty"` + Pid int32 `protobuf:"varint,16,opt,name=pid,proto3" json:"pid,omitempty"` + RunType string `protobuf:"bytes,17,opt,name=run_type,json=runType,proto3" json:"run_type,omitempty"` + ScheduleId string `protobuf:"bytes,18,opt,name=schedule_id,json=scheduleId,proto3" json:"schedule_id,omitempty"` + Type string `protobuf:"bytes,19,opt,name=type,proto3" json:"type,omitempty"` +} + +func (x *Task) Reset() { + *x = Task{} + if protoimpl.UnsafeEnabled { + mi := &file_models_task_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Task) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Task) ProtoMessage() {} + +func (x *Task) ProtoReflect() protoreflect.Message { + mi := &file_models_task_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Task.ProtoReflect.Descriptor instead. +func (*Task) Descriptor() ([]byte, []int) { + return file_models_task_proto_rawDescGZIP(), []int{0} +} + +func (x *Task) GetXId() string { + if x != nil { + return x.XId + } + return "" +} + +func (x *Task) GetSpiderId() string { + if x != nil { + return x.SpiderId + } + return "" +} + +func (x *Task) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *Task) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + +func (x *Task) GetCmd() string { + if x != nil { + return x.Cmd + } + return "" +} + +func (x *Task) GetParam() string { + if x != nil { + return x.Param + } + return "" +} + +func (x *Task) GetError() string { + if x != nil { + return x.Error + } + return "" +} + +func (x *Task) GetPid() int32 { + if x != nil { + return x.Pid + } + return 0 +} + +func (x *Task) GetRunType() string { + if x != nil { + return x.RunType + } + return "" +} + +func (x *Task) GetScheduleId() string { + if x != nil { + return x.ScheduleId + } + return "" +} + +func (x *Task) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +var File_models_task_proto protoreflect.FileDescriptor + +var file_models_task_proto_rawDesc = []byte{ + 0x0a, 0x11, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x73, 0x2f, 0x74, 0x61, 0x73, 0x6b, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, 0x72, 0x70, 0x63, 0x22, 0x85, 0x02, 0x0a, 0x04, 0x54, 0x61, + 0x73, 0x6b, 0x12, 0x0f, 0x0a, 0x03, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x70, 0x69, 0x64, 0x65, 0x72, 0x5f, 0x69, 0x64, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x70, 0x69, 0x64, 0x65, 0x72, 0x49, 0x64, + 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, + 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x6d, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x63, 0x6d, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x18, 0x09, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, + 0x10, 0x0a, 0x03, 0x70, 0x69, 0x64, 0x18, 0x10, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x70, 0x69, + 0x64, 0x12, 0x19, 0x0a, 0x08, 0x72, 0x75, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x11, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x72, 0x75, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, + 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x12, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, + 0x65, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x33, +} + +var ( + file_models_task_proto_rawDescOnce sync.Once + file_models_task_proto_rawDescData = file_models_task_proto_rawDesc +) + +func file_models_task_proto_rawDescGZIP() []byte { + file_models_task_proto_rawDescOnce.Do(func() { + file_models_task_proto_rawDescData = protoimpl.X.CompressGZIP(file_models_task_proto_rawDescData) + }) + return file_models_task_proto_rawDescData +} + +var file_models_task_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_models_task_proto_goTypes = []interface{}{ + (*Task)(nil), // 0: grpc.Task +} +var file_models_task_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_models_task_proto_init() } +func file_models_task_proto_init() { + if File_models_task_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_models_task_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Task); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_models_task_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_models_task_proto_goTypes, + DependencyIndexes: file_models_task_proto_depIdxs, + MessageInfos: file_models_task_proto_msgTypes, + }.Build() + File_models_task_proto = out.File + file_models_task_proto_rawDesc = nil + file_models_task_proto_goTypes = nil + file_models_task_proto_depIdxs = nil +} diff --git a/grpc/task_service.pb.go b/grpc/task_service.pb.go new file mode 100644 index 00000000..7ea57caa --- /dev/null +++ b/grpc/task_service.pb.go @@ -0,0 +1,95 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.25.0 +// protoc v3.20.1 +// source: services/task_service.proto + +package grpc + +import ( + proto "github.com/golang/protobuf/proto" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// This is a compile-time assertion that a sufficiently up-to-date version +// of the legacy proto package is being used. +const _ = proto.ProtoPackageIsVersion4 + +var File_services_task_service_proto protoreflect.FileDescriptor + +var file_services_task_service_proto_rawDesc = []byte{ + 0x0a, 0x1b, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x2f, 0x74, 0x61, 0x73, 0x6b, 0x5f, + 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x67, + 0x72, 0x70, 0x63, 0x1a, 0x14, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x72, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x15, 0x65, 0x6e, 0x74, 0x69, 0x74, + 0x79, 0x2f, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x1b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x2f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, + 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0xa2, 0x01, + 0x0a, 0x0b, 0x54, 0x61, 0x73, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x34, 0x0a, + 0x09, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x13, 0x2e, 0x67, 0x72, 0x70, + 0x63, 0x2e, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, + 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x28, 0x01, 0x12, 0x28, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x0d, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0e, 0x2e, 0x67, 0x72, + 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x33, 0x0a, + 0x10, 0x53, 0x65, 0x6e, 0x64, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x12, 0x0d, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x0e, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x42, 0x08, 0x5a, 0x06, 0x2e, 0x3b, 0x67, 0x72, 0x70, 0x63, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var file_services_task_service_proto_goTypes = []interface{}{ + (*StreamMessage)(nil), // 0: grpc.StreamMessage + (*Request)(nil), // 1: grpc.Request + (*Response)(nil), // 2: grpc.Response +} +var file_services_task_service_proto_depIdxs = []int32{ + 0, // 0: grpc.TaskService.Subscribe:input_type -> grpc.StreamMessage + 1, // 1: grpc.TaskService.Fetch:input_type -> grpc.Request + 1, // 2: grpc.TaskService.SendNotification:input_type -> grpc.Request + 2, // 3: grpc.TaskService.Subscribe:output_type -> grpc.Response + 2, // 4: grpc.TaskService.Fetch:output_type -> grpc.Response + 2, // 5: grpc.TaskService.SendNotification:output_type -> grpc.Response + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_services_task_service_proto_init() } +func file_services_task_service_proto_init() { + if File_services_task_service_proto != nil { + return + } + file_entity_request_proto_init() + file_entity_response_proto_init() + file_entity_stream_message_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_services_task_service_proto_rawDesc, + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_services_task_service_proto_goTypes, + DependencyIndexes: file_services_task_service_proto_depIdxs, + }.Build() + File_services_task_service_proto = out.File + file_services_task_service_proto_rawDesc = nil + file_services_task_service_proto_goTypes = nil + file_services_task_service_proto_depIdxs = nil +} diff --git a/grpc/task_service_grpc.pb.go b/grpc/task_service_grpc.pb.go new file mode 100644 index 00000000..6afa1226 --- /dev/null +++ b/grpc/task_service_grpc.pb.go @@ -0,0 +1,212 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.2.0 +// - protoc v3.20.1 +// source: services/task_service.proto + +package grpc + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.32.0 or later. +const _ = grpc.SupportPackageIsVersion7 + +// TaskServiceClient is the client API for TaskService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type TaskServiceClient interface { + Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) + Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) + SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) +} + +type taskServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewTaskServiceClient(cc grpc.ClientConnInterface) TaskServiceClient { + return &taskServiceClient{cc} +} + +func (c *taskServiceClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (TaskService_SubscribeClient, error) { + stream, err := c.cc.NewStream(ctx, &TaskService_ServiceDesc.Streams[0], "/grpc.TaskService/Subscribe", opts...) + if err != nil { + return nil, err + } + x := &taskServiceSubscribeClient{stream} + return x, nil +} + +type TaskService_SubscribeClient interface { + Send(*StreamMessage) error + CloseAndRecv() (*Response, error) + grpc.ClientStream +} + +type taskServiceSubscribeClient struct { + grpc.ClientStream +} + +func (x *taskServiceSubscribeClient) Send(m *StreamMessage) error { + return x.ClientStream.SendMsg(m) +} + +func (x *taskServiceSubscribeClient) CloseAndRecv() (*Response, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(Response) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *taskServiceClient) Fetch(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.TaskService/Fetch", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskServiceClient) SendNotification(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) { + out := new(Response) + err := c.cc.Invoke(ctx, "/grpc.TaskService/SendNotification", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// TaskServiceServer is the server API for TaskService service. +// All implementations must embed UnimplementedTaskServiceServer +// for forward compatibility +type TaskServiceServer interface { + Subscribe(TaskService_SubscribeServer) error + Fetch(context.Context, *Request) (*Response, error) + SendNotification(context.Context, *Request) (*Response, error) + mustEmbedUnimplementedTaskServiceServer() +} + +// UnimplementedTaskServiceServer must be embedded to have forward compatible implementations. +type UnimplementedTaskServiceServer struct { +} + +func (UnimplementedTaskServiceServer) Subscribe(TaskService_SubscribeServer) error { + return status.Errorf(codes.Unimplemented, "method Subscribe not implemented") +} +func (UnimplementedTaskServiceServer) Fetch(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method Fetch not implemented") +} +func (UnimplementedTaskServiceServer) SendNotification(context.Context, *Request) (*Response, error) { + return nil, status.Errorf(codes.Unimplemented, "method SendNotification not implemented") +} +func (UnimplementedTaskServiceServer) mustEmbedUnimplementedTaskServiceServer() {} + +// UnsafeTaskServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TaskServiceServer will +// result in compilation errors. +type UnsafeTaskServiceServer interface { + mustEmbedUnimplementedTaskServiceServer() +} + +func RegisterTaskServiceServer(s grpc.ServiceRegistrar, srv TaskServiceServer) { + s.RegisterService(&TaskService_ServiceDesc, srv) +} + +func _TaskService_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TaskServiceServer).Subscribe(&taskServiceSubscribeServer{stream}) +} + +type TaskService_SubscribeServer interface { + SendAndClose(*Response) error + Recv() (*StreamMessage, error) + grpc.ServerStream +} + +type taskServiceSubscribeServer struct { + grpc.ServerStream +} + +func (x *taskServiceSubscribeServer) SendAndClose(m *Response) error { + return x.ServerStream.SendMsg(m) +} + +func (x *taskServiceSubscribeServer) Recv() (*StreamMessage, error) { + m := new(StreamMessage) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func _TaskService_Fetch_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskServiceServer).Fetch(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.TaskService/Fetch", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskServiceServer).Fetch(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +func _TaskService_SendNotification_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(Request) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskServiceServer).SendNotification(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpc.TaskService/SendNotification", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskServiceServer).SendNotification(ctx, req.(*Request)) + } + return interceptor(ctx, in, info, handler) +} + +// TaskService_ServiceDesc is the grpc.ServiceDesc for TaskService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TaskService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "grpc.TaskService", + HandlerType: (*TaskServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Fetch", + Handler: _TaskService_Fetch_Handler, + }, + { + MethodName: "SendNotification", + Handler: _TaskService_SendNotification_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "Subscribe", + Handler: _TaskService_Subscribe_Handler, + ClientStreams: true, + }, + }, + Metadata: "services/task_service.proto", +}