crawlab/core/controllers/sync.go

package controllers

import (
	"context"
	"path/filepath"
	"sync/atomic"

	"github.com/crawlab-team/crawlab/core/entity"
	"github.com/crawlab-team/crawlab/core/utils"
	"github.com/gin-gonic/gin"
	"github.com/juju/errors"
	"golang.org/x/sync/semaphore"
)

var (
	syncDownloadSemaphore = semaphore.NewWeighted(utils.GetSyncDownloadMaxConcurrency())
	syncDownloadInFlight  int64
)

func GetSyncScan(c *gin.Context) (response *Response[entity.FsFileInfoMap], err error) {
	workspacePath := utils.GetWorkspace()
	dirPath := filepath.Join(workspacePath, c.Param("id"), c.Param("path"))
	files, err := utils.ScanDirectory(dirPath)
	if err != nil {
		logger.Warnf("sync scan failed id=%s path=%s: %v", c.Param("id"), c.Param("path"), err)
		return GetErrorResponse[entity.FsFileInfoMap](err)
	}
	return GetDataResponse(files)
}

func GetSyncDownload(c *gin.Context) (err error) {
	ctx := c.Request.Context()
	if ctx == nil {
		ctx = context.Background()
	}

	if err := syncDownloadSemaphore.Acquire(ctx, 1); err != nil {
		logger.Warnf("failed to acquire sync download slot for id=%s path=%s: %v", c.Param("id"), c.Query("path"), err)
		return errors.Annotate(err, "acquire sync download slot")
	}
	current := atomic.AddInt64(&syncDownloadInFlight, 1)
	logger.Debugf("sync download in-flight=%d id=%s path=%s", current, c.Param("id"), c.Query("path"))
	defer func() {
		newVal := atomic.AddInt64(&syncDownloadInFlight, -1)
		logger.Debugf("sync download completed in-flight=%d id=%s path=%s", newVal, c.Param("id"), c.Query("path"))
		syncDownloadSemaphore.Release(1)
	}()

	workspacePath := utils.GetWorkspace()
	filePath := filepath.Join(workspacePath, c.Param("id"), c.Query("path"))
	if !utils.Exists(filePath) {
		return errors.NotFoundf("file not exists: %s", filePath)
	}
	c.File(filePath)
	return nil
}