git-sync/crawlab
1
0
Fork 0
mirror of https://github.com/crawlab-team/crawlab.git synced 2026-01-21 17:21:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

fix: unable to sync files from spiders created from sub-folder of git repo

Browse Source
This commit is contained in:
Marvin Zhang
2024-07-05 12:57:09 +08:00
parent f4025a982d
commit 059711a533
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
core/task/handler/runner_v2.go
View File

@@ -18,7 +18,7 @@ import (
service2 "github.com/crawlab-team/crawlab/core/models/service"
"github.com/crawlab-team/crawlab/core/sys_exec"
"github.com/crawlab-team/crawlab/core/utils"
grpc "github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/grpc"
"github.com/crawlab-team/crawlab/trace"
"github.com/shirou/gopsutil/process"
"github.com/spf13/viper"
@@ -414,7 +414,7 @@ func (r *RunnerV2) syncFiles() (err error) {
}
} else {
log.Infof("File needs to be synchronized: %s", path)
_err := r.downloadFile(masterURL+"/download?path="+path, filepath.Join(r.cwd, path), masterFile)
_err := r.downloadFile(masterURL+"/download?path="+filepath.Join(workingDir, path), filepath.Join(r.cwd, path), masterFile)
if _err != nil {
log.Errorf("Error downloading file: %v", _err)
err = errors.Join(err, _err)
@@ -441,6 +441,10 @@ func (r *RunnerV2) downloadFile(url string, filePath string, fileInfo *entity.Fs
log.Errorf("Error getting file response: %v", err)
return err
}
if resp.StatusCode != http.StatusOK {
log.Errorf("Error downloading file: %s", resp.Status)
return errors.New(resp.Status)
}
defer resp.Body.Close()
// create directory if not exists