git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-26 17:49:15 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix(spider): update node selection to use active nodes instead of all nodes

Browse Source 
fix(spider): optimize form update logic to watch specific fields for changes
fix(grpc): adjust sync request ID handling for git and regular spiders
This commit is contained in:
Marvin Zhang
2025-12-02 11:42:29 +08:00
parent 97ab39119c
commit b2ff8baed8
4 changed files with 38 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
core/task/handler/runner_sync_grpc.go
View File

@@ -33,9 +33,19 @@ func (r *Runner) syncFilesGRPC() (err error) {
return err
}
// Determine the ID to use for sync request
// For git spiders, use GitId (where files are stored)
// For regular spiders, use SpiderId
var syncId string
if r.s.GitId.IsZero() {
syncId = r.s.Id.Hex()
} else {
syncId = r.s.GitId.Hex()
}
// Prepare request
req := &grpc2.FileSyncRequest{
SpiderId: r.s.Id.Hex(),
SpiderId: syncId,
Path: workingDir,
NodeKey: utils.GetNodeKey(),
}
@@ -149,7 +159,13 @@ func (r *Runner) syncFilesGRPC() (err error) {
}
if needsDownload {
if err := r.downloadFileGRPC(syncClient, r.s.Id.Hex(), path); err != nil {
// For git spiders with sub-folders, prepend GitRootPath to the download path
// because the server stores files at workspace/{git_id}/{git_root_path}/...
downloadPath := path
if workingDir != "" {
downloadPath = filepath.Join(workingDir, path)
}
if err := r.downloadFileGRPC(syncClient, syncId, downloadPath); err != nil {
r.Errorf("error downloading file %s: %v", path, err)
return err
}