feat: added modules

core/task/handler/runner_test.go

@@ -0,0 +1,103 @@
+package handler
+
+import (
+	"github.com/crawlab-team/crawlab/core/models/models"
+	"github.com/google/uuid"
+	"github.com/spf13/viper"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+type MockRunner struct {
+	mock.Mock
+	Runner
+}
+
+func (m *MockRunner) downloadFile(url string, filePath string) error {
+	args := m.Called(url, filePath)
+	return args.Error(0)
+}
+
+func newMockRunner() *MockRunner {
+	r := &MockRunner{}
+	r.s = &models.Spider{}
+	return r
+}
+
+func TestSyncFiles_SuccessWithDummyFiles(t *testing.T) {
+	// Create a test server that responds with a list of files
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/scan") {
+			w.Write([]byte(`{"file1.txt":{"path": "file1.txt", "hash": "hash1"}, "file2.txt":{"path": "file2.txt", "hash": "hash2"}}`))
+			return
+		}
+
+		if strings.HasSuffix(r.URL.Path, "/download") {
+			w.Write([]byte("file content"))
+			return
+		}
+	}))
+	defer ts.Close()
+
+	// Create a mock runner
+	r := newMockRunner()
+	r.On("downloadFile", mock.Anything, mock.Anything).Return(nil)
+
+	// Set the master URL to the test server URL
+	viper.Set("api.endpoint", ts.URL)
+
+	localPath := filepath.Join(os.TempDir(), uuid.New().String())
+	os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm)
+	defer os.RemoveAll(localPath)
+	viper.Set("workspace", localPath)
+
+	// Call the method under test
+	err := r.syncFiles()
+	assert.NoError(t, err)
+
+	// Assert that the files were downloaded
+	assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file1.txt"))
+	assert.FileExists(t, filepath.Join(localPath, r.s.GetId().Hex(), "file2.txt"))
+}
+
+func TestSyncFiles_DeletesFilesNotOnMaster(t *testing.T) {
+	// Create a test server that responds with an empty list of files
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasSuffix(r.URL.Path, "/scan") {
+			w.Write([]byte(`{}`))
+			return
+		}
+	}))
+	defer ts.Close()
+
+	// Create a mock runner
+	r := newMockRunner()
+	r.On("downloadFile", mock.Anything, mock.Anything).Return(nil)
+
+	// Set the master URL to the test server URL
+	viper.Set("api.endpoint", ts.URL)
+
+	localPath := filepath.Join(os.TempDir(), uuid.New().String())
+	os.MkdirAll(filepath.Join(localPath, r.s.GetId().Hex()), os.ModePerm)
+	defer os.RemoveAll(localPath)
+	viper.Set("workspace", localPath)
+
+	// Create a dummy file that should be deleted
+	dummyFilePath := filepath.Join(localPath, r.s.GetId().Hex(), "dummy.txt")
+	dummyFile, _ := os.Create(dummyFilePath)
+	dummyFile.Close()
+
+	// Call the method under test
+	err := r.syncFiles()
+	assert.NoError(t, err)
+
+	// Assert that the dummy file was deleted
+	assert.NoFileExists(t, dummyFilePath)
+}