mirror of
https://github.com/crawlab-team/crawlab.git
synced 2026-01-21 17:21:09 +01:00
feat: added modules
This commit is contained in:
Marvin Zhang
9
fs/.editorconfig
Normal file
9
fs/.editorconfig
Normal file
@@ -0,0 +1,9 @@
|
||||
root = true
|
||||
|
||||
[*.go]
|
||||
charset = utf-8
|
||||
end_of_line = lf
|
||||
indent_size = 4
|
||||
indent_style = tab
|
||||
insert_final_newline = true
|
||||
trim_trailing_whitespace = true
|
||||
54
fs/.github/workflows/test.yml
vendored
Normal file
54
fs/.github/workflows/test.yml
vendored
Normal file
@@ -0,0 +1,54 @@
|
||||
name: Test and coverage
|
||||
|
||||
on: [ push, pull_request ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Setup Go environment
|
||||
uses: actions/setup-go@v2.1.3
|
||||
with:
|
||||
# The Go version to download (if necessary) and use. Supports semver spec and ranges.
|
||||
go-version: 1.15
|
||||
|
||||
# - name: Download Binary Files
|
||||
# run: |
|
||||
# mkdir -p $GITHUB_WORKSPACE/seaweedfs
|
||||
# curl https://github.com/chrislusf/seaweedfs/releases/download/2.48/linux_amd64.tar.gz -o $GITHUB_WORKSPACE/seaweedfs/linux_amd64.tar.gz
|
||||
# cd $GITHUB_WORKSPACE/seaweedfs
|
||||
# ls -l
|
||||
# tar -zxf linux_amd64.tar.gz
|
||||
- name: Download Binary Files
|
||||
uses: fabriciobastian/download-release-asset-action@v1.0.6
|
||||
with:
|
||||
# A specific release version. Defaults to latest
|
||||
version: 2.48 # default is latest
|
||||
# Relative path to the repository in the format user/repo e.g.: myuser/my-repository
|
||||
repository: chrislusf/seaweedfs # default is
|
||||
# The name of the asset to download from the release
|
||||
file: linux_amd64.tar.gz
|
||||
# Path to the directory where to download the asset
|
||||
out: seaweedfs # optional, default is .
|
||||
|
||||
- name: Extract Binary Files
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/seaweedfs
|
||||
tar -zxf linux_amd64.tar.gz
|
||||
|
||||
- name: Validate Binary Files
|
||||
run: |
|
||||
cd $GITHUB_WORKSPACE/seaweedfs
|
||||
ls -l weed
|
||||
|
||||
- name: Run Tests
|
||||
run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic -coverpkg github.com/crawlab-team/crawlab/fs
|
||||
|
||||
- name: Codecov
|
||||
uses: codecov/codecov-action@v1.5.0
|
||||
with:
|
||||
# Repository upload token - get it from codecov.io. Required only for private repositories
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
# Comma-separated list of files to upload
|
||||
23
fs/.gitignore
vendored
Normal file
23
fs/.gitignore
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
vendor/
|
||||
tmp/
|
||||
.idea/
|
||||
.DS_Store
|
||||
coverage.txt
|
||||
|
||||
filerldb2
|
||||
seaweedfs
|
||||
*.txt
|
||||
8
fs/Dockerfile
Normal file
8
fs/Dockerfile
Normal file
@@ -0,0 +1,8 @@
|
||||
FROM golang:1.15
|
||||
|
||||
WORKDIR /app
|
||||
ADD ./go.mod /app
|
||||
ADD ./go.sum /app
|
||||
RUN go mod download
|
||||
|
||||
CMD ["sh", "./bin/test.sh"]
|
||||
29
fs/LICENSE
Normal file
29
fs/LICENSE
Normal file
@@ -0,0 +1,29 @@
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2020, Crawlab Team
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
5
fs/README.md
Normal file
5
fs/README.md
Normal file
@@ -0,0 +1,5 @@
|
||||
# crawlab-fs
|
||||
|
||||
[](https://codecov.io/gh/crawlab-team/crawlab-fs)
|
||||
|
||||
Backend file system module for Crawlab
|
||||
24
fs/bin/start.sh
Executable file
24
fs/bin/start.sh
Executable file
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
if [ -e ./tmp ]; then
|
||||
:
|
||||
else
|
||||
mkdir ./tmp
|
||||
fi
|
||||
|
||||
if [ -x /usr/local/bin/weed ]; then
|
||||
weed server \
|
||||
-dir ./tmp \
|
||||
-master.dir ./tmp \
|
||||
-volume.dir.idx ./tmp \
|
||||
-ip localhost \
|
||||
-ip.bind 0.0.0.0 \
|
||||
-filer
|
||||
else
|
||||
./seaweedfs/weed server \
|
||||
-dir ./tmp \
|
||||
-master.dir ./tmp \
|
||||
-volume.dir.idx ./tmp \
|
||||
-ip localhost \
|
||||
-ip.bind 0.0.0.0 \
|
||||
-filer
|
||||
fi
|
||||
3
fs/bin/stop.sh
Executable file
3
fs/bin/stop.sh
Executable file
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
|
||||
kill -9 `ps axu|grep weed|grep -v grep|awk '{print \$2}'|xargs`
|
||||
19
fs/constants.go
Normal file
19
fs/constants.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package fs
|
||||
|
||||
import "os"
|
||||
|
||||
const (
|
||||
FilerResponseNotFoundErrorMessage = "response status code: 404"
|
||||
FilerStatusNotFoundErrorMessage = "Status:404 Not Found"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultDirMode = os.FileMode(0766)
|
||||
DefaultFileMode = os.FileMode(0666)
|
||||
)
|
||||
|
||||
const (
|
||||
MethodUpdateFile = "update-file"
|
||||
MethodUploadFile = "upload-file"
|
||||
MethodUploadDir = "upload-dir"
|
||||
)
|
||||
13
fs/docker-compose.yml
Normal file
13
fs/docker-compose.yml
Normal file
@@ -0,0 +1,13 @@
|
||||
version: '2'
|
||||
|
||||
services:
|
||||
server:
|
||||
image: chrislusf/seaweedfs # use a remote image
|
||||
container_name: seaweedfs-server
|
||||
restart: always
|
||||
# command: "server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer -encryptVolumeData"
|
||||
command: "server -dir /data -master.dir /data -volume.dir.idx /data -ip localhost -ip.bind 0.0.0.0 -filer"
|
||||
ports:
|
||||
- 8888:8888
|
||||
#volumes:
|
||||
# - /data/seaweedfs:/data
|
||||
5
fs/errors.go
Normal file
5
fs/errors.go
Normal file
@@ -0,0 +1,5 @@
|
||||
package fs
|
||||
|
||||
import "errors"
|
||||
|
||||
var ErrorFsNotExists = errors.New("not exists")
|
||||
17
fs/go.mod
Normal file
17
fs/go.mod
Normal file
@@ -0,0 +1,17 @@
|
||||
module github.com/crawlab-team/crawlab/fs
|
||||
|
||||
go 1.22
|
||||
|
||||
replace (
|
||||
github.com/crawlab-team/crawlab/trace => ../trace
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/apex/log v1.9.0
|
||||
github.com/cenkalti/backoff/v4 v4.1.0
|
||||
github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/stretchr/testify v1.6.1
|
||||
)
|
||||
102
fs/go.sum
Normal file
102
fs/go.sum
Normal file
@@ -0,0 +1,102 @@
|
||||
github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0=
|
||||
github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA=
|
||||
github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo=
|
||||
github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE=
|
||||
github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys=
|
||||
github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
|
||||
github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
|
||||
github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc=
|
||||
github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
|
||||
github.com/crawlab-team/crawlab/trace v0.1.0 h1:uCqfdqNfb+NwqdkQrBkcYfQ9iqGJ76MbPw1wK8n7xGg=
|
||||
github.com/crawlab-team/crawlab/trace v0.1.0/go.mod h1:LcWyn68HoT+d29CHM8L41pFHxsAcBMF1xjqJmWdyFh8=
|
||||
github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936 h1:c4SgTj2baDqD2UYa1eCpj3ukOF3mXOjvOCP4cWwgfyw=
|
||||
github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA=
|
||||
github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69 h1:qPLsh2aWqI5HioWBymzQirt+HQxfRgd7BSoOqfN33Q0=
|
||||
github.com/crawlab-team/goseaweedfs v0.6.0-beta.20211101.1936.0.20220912021203-dfee5f74dd69/go.mod h1:u+rwfqb0rnYllTLjCctE/z1Yp+TC8L+CbbWH8E2NstA=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ=
|
||||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
|
||||
github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/linxGnu/gumble v1.0.0 h1:OAJud8Hy4rmV9I5p/KTRiVpwwklMTd9Ankza3Mz7a4M=
|
||||
github.com/linxGnu/gumble v1.0.0/go.mod h1:iyhNJpBHvJ0q2Hr41iiZRJyj6LLF47i2a9C9zLiucVY=
|
||||
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e h1:9MlwzLdW7QSDrhDjFlsEYmxpFyIoXmYRon3dt0io31k=
|
||||
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
|
||||
github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
|
||||
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
|
||||
github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
|
||||
github.com/scryner/lfreequeue v0.0.0-20121212074822-473f33702129/go.mod h1:0OrdloYlIayHGsgKYlwEnmdrPWmuYtbdS6Dm71PprFM=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM=
|
||||
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
|
||||
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0=
|
||||
github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
|
||||
github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk=
|
||||
github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc=
|
||||
github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0=
|
||||
github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao=
|
||||
github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4=
|
||||
github.com/valyala/fastrand v1.0.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
|
||||
github.com/ztrue/tracerr v0.3.0 h1:lDi6EgEYhPYPnKcjsYzmWw4EkFEoA/gfe+I9Y5f+h6Y=
|
||||
github.com/ztrue/tracerr v0.3.0/go.mod h1:qEalzze4VN9O8tnhBXScfCrmoJo10o8TN5ciKjm6Mww=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
31
fs/interface.go
Normal file
31
fs/interface.go
Normal file
@@ -0,0 +1,31 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"github.com/crawlab-team/goseaweedfs"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Manager interface {
|
||||
Init() (err error)
|
||||
Close() (err error)
|
||||
ListDir(remotePath string, isRecursive bool) (files []goseaweedfs.FilerFileInfo, err error)
|
||||
UploadFile(localPath, remotePath string, args ...interface{}) (err error)
|
||||
UploadDir(localPath, remotePath string, args ...interface{}) (err error)
|
||||
DownloadFile(remotePath, localPath string, args ...interface{}) (err error)
|
||||
DownloadDir(remotePath, localPath string, args ...interface{}) (err error)
|
||||
DeleteFile(remotePath string) (err error)
|
||||
DeleteDir(remotePath string) (err error)
|
||||
SyncLocalToRemote(localPath, remotePath string, args ...interface{}) (err error)
|
||||
SyncRemoteToLocal(remotePath, localPath string, args ...interface{}) (err error)
|
||||
GetFile(remotePath string, args ...interface{}) (data []byte, err error)
|
||||
GetFileInfo(remotePath string) (file *goseaweedfs.FilerFileInfo, err error)
|
||||
UpdateFile(remotePath string, data []byte, args ...interface{}) (err error)
|
||||
Exists(remotePath string, args ...interface{}) (ok bool, err error)
|
||||
SetFilerUrl(url string)
|
||||
SetFilerAuthKey(authKey string)
|
||||
SetTimeout(timeout time.Duration)
|
||||
SetWorkerNum(num int)
|
||||
SetRetryInterval(interval time.Duration)
|
||||
SetRetryNum(num int)
|
||||
SetMaxQps(qps int)
|
||||
}
|
||||
111
fs/lib/copy/copy.go
Normal file
111
fs/lib/copy/copy.go
Normal file
@@ -0,0 +1,111 @@
|
||||
package copy
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func CopyDirectory(scrDir, dest string) error {
|
||||
entries, err := ioutil.ReadDir(scrDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
sourcePath := filepath.Join(scrDir, entry.Name())
|
||||
destPath := filepath.Join(dest, entry.Name())
|
||||
|
||||
fileInfo, err := os.Stat(sourcePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
|
||||
if !ok {
|
||||
return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
|
||||
}
|
||||
|
||||
switch fileInfo.Mode() & os.ModeType {
|
||||
case os.ModeDir:
|
||||
if err := CreateIfNotExists(destPath, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := CopyDirectory(sourcePath, destPath); err != nil {
|
||||
return err
|
||||
}
|
||||
case os.ModeSymlink:
|
||||
if err := CopySymLink(sourcePath, destPath); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
if err := Copy(sourcePath, destPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isSymlink := entry.Mode()&os.ModeSymlink != 0
|
||||
if !isSymlink {
|
||||
if err := os.Chmod(destPath, entry.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func Copy(srcFile, dstFile string) error {
|
||||
out, err := os.Create(dstFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer out.Close()
|
||||
|
||||
in, err := os.Open(srcFile)
|
||||
defer in.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func Exists(filePath string) bool {
|
||||
if _, err := os.Stat(filePath); os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func CreateIfNotExists(dir string, perm os.FileMode) error {
|
||||
if Exists(dir) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(dir, perm); err != nil {
|
||||
return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func CopySymLink(source, dest string) error {
|
||||
link, err := os.Readlink(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Symlink(link, dest)
|
||||
}
|
||||
47
fs/options.go
Normal file
47
fs/options.go
Normal file
@@ -0,0 +1,47 @@
|
||||
package fs
|
||||
|
||||
import "time"
|
||||
|
||||
type Option func(m Manager)
|
||||
|
||||
func WithFilerUrl(url string) Option {
|
||||
return func(m Manager) {
|
||||
m.SetFilerUrl(url)
|
||||
}
|
||||
}
|
||||
|
||||
func WithFilerAuthKey(authKey string) Option {
|
||||
return func(m Manager) {
|
||||
m.SetFilerAuthKey(authKey)
|
||||
}
|
||||
}
|
||||
|
||||
func WithTimeout(timeout time.Duration) Option {
|
||||
return func(m Manager) {
|
||||
m.SetTimeout(timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func WithWorkerNum(num int) Option {
|
||||
return func(m Manager) {
|
||||
m.SetWorkerNum(num)
|
||||
}
|
||||
}
|
||||
|
||||
func WithRetryInterval(interval time.Duration) Option {
|
||||
return func(m Manager) {
|
||||
m.SetRetryInterval(interval)
|
||||
}
|
||||
}
|
||||
|
||||
func WithRetryNum(num int) Option {
|
||||
return func(m Manager) {
|
||||
m.SetRetryNum(num)
|
||||
}
|
||||
}
|
||||
|
||||
func WithMaxQps(qps int) Option {
|
||||
return func(m Manager) {
|
||||
m.SetMaxQps(qps)
|
||||
}
|
||||
}
|
||||
722
fs/seaweedfs_manager.go
Normal file
722
fs/seaweedfs_manager.go
Normal file
@@ -0,0 +1,722 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/crawlab-team/goseaweedfs"
|
||||
"github.com/emirpasic/gods/queues/linkedlistqueue"
|
||||
"github.com/google/uuid"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type seaweedFsManagerFn func(params seaweedFsManagerParams) (res seaweedFsManagerResults)
|
||||
|
||||
type seaweedFsManagerHandle struct {
|
||||
params seaweedFsManagerParams
|
||||
fn seaweedFsManagerFn
|
||||
resChan chan seaweedFsManagerResults
|
||||
}
|
||||
|
||||
type seaweedFsManagerParams struct {
|
||||
localPath string
|
||||
remotePath string
|
||||
isRecursive bool
|
||||
collection string
|
||||
ttl string
|
||||
urlValues url.Values
|
||||
data []byte
|
||||
}
|
||||
|
||||
type seaweedFsManagerResults struct {
|
||||
files []goseaweedfs.FilerFileInfo
|
||||
file *goseaweedfs.FilerFileInfo
|
||||
data []byte
|
||||
ok bool
|
||||
err error
|
||||
}
|
||||
|
||||
type SeaweedFsManager struct {
|
||||
// settings variables
|
||||
filerUrl string
|
||||
timeout time.Duration
|
||||
authKey string
|
||||
workerNum int
|
||||
retryNum uint64
|
||||
retryInterval time.Duration
|
||||
maxQps int
|
||||
|
||||
// internals
|
||||
f *goseaweedfs.Filer
|
||||
q *linkedlistqueue.Queue
|
||||
cr int
|
||||
ch chan seaweedFsManagerHandle
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) Init() (err error) {
|
||||
// filer options
|
||||
var filerOpts []goseaweedfs.FilerOption
|
||||
|
||||
// auth key
|
||||
if m.authKey != "" {
|
||||
filerOpts = append(filerOpts, goseaweedfs.WithFilerAuthKey(m.authKey))
|
||||
}
|
||||
|
||||
// handle channel
|
||||
m.ch = make(chan seaweedFsManagerHandle, m.workerNum)
|
||||
|
||||
// filer instance
|
||||
m.f, err = goseaweedfs.NewFiler(m.filerUrl, &http.Client{Timeout: m.timeout}, filerOpts...)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
// start async
|
||||
go m.start()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) Close() (err error) {
|
||||
m.closed = true
|
||||
if err := m.f.Close(); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) ListDir(remotePath string, isRecursive bool) (files []goseaweedfs.FilerFileInfo, err error) {
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
isRecursive: isRecursive,
|
||||
}
|
||||
res := m.process(params, m.listDir)
|
||||
return res.files, res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) ListDirRecursive(remotePath string) (files []goseaweedfs.FilerFileInfo, err error) {
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
}
|
||||
res := m.process(params, m.listDirRecursive)
|
||||
return res.files, res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) UploadFile(localPath, remotePath string, args ...interface{}) (err error) {
|
||||
localPath, err = filepath.Abs(localPath)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
collection, ttl := getCollectionAndTtlFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
localPath: localPath,
|
||||
remotePath: remotePath,
|
||||
collection: collection,
|
||||
ttl: ttl,
|
||||
}
|
||||
res := m.process(params, m.uploadFile)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) UploadDir(localPath, remotePath string, args ...interface{}) (err error) {
|
||||
localPath, err = filepath.Abs(localPath)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
collection, ttl := getCollectionAndTtlFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
localPath: localPath,
|
||||
remotePath: remotePath,
|
||||
collection: collection,
|
||||
ttl: ttl,
|
||||
}
|
||||
res := m.process(params, m.uploadDir)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) DownloadFile(remotePath, localPath string, args ...interface{}) (err error) {
|
||||
localPath, err = filepath.Abs(localPath)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
collection, ttl := getCollectionAndTtlFromArgs(args...)
|
||||
urlValues := getUrlValuesFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
localPath: localPath,
|
||||
remotePath: remotePath,
|
||||
collection: collection,
|
||||
ttl: ttl,
|
||||
urlValues: urlValues,
|
||||
}
|
||||
res := m.process(params, m.downloadFile)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) DownloadDir(remotePath, localPath string, args ...interface{}) (err error) {
|
||||
localPath, err = filepath.Abs(localPath)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
}
|
||||
res := m.process(params, m.downloadDir)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) DeleteFile(remotePath string) (err error) {
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
}
|
||||
res := m.process(params, m.deleteFile)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) DeleteDir(remotePath string) (err error) {
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
}
|
||||
res := m.process(params, m.deleteDir)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SyncLocalToRemote(localPath, remotePath string, args ...interface{}) (err error) {
|
||||
localPath, err = filepath.Abs(localPath)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
collection, ttl := getCollectionAndTtlFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
localPath: localPath,
|
||||
remotePath: remotePath,
|
||||
collection: collection,
|
||||
ttl: ttl,
|
||||
}
|
||||
res := m.process(params, m.syncLocalToRemote)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SyncRemoteToLocal(remotePath, localPath string, args ...interface{}) (err error) {
|
||||
collection, ttl := getCollectionAndTtlFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
localPath: localPath,
|
||||
remotePath: remotePath,
|
||||
collection: collection,
|
||||
ttl: ttl,
|
||||
}
|
||||
res := m.process(params, m.syncRemoteToLocal)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) GetFile(remotePath string, args ...interface{}) (data []byte, err error) {
|
||||
urlValues := getUrlValuesFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
urlValues: urlValues,
|
||||
}
|
||||
res := m.process(params, m.getFile)
|
||||
return res.data, res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) GetFileInfo(remotePath string) (file *goseaweedfs.FilerFileInfo, err error) {
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
}
|
||||
res := m.process(params, m.getFileInfo)
|
||||
return res.file, res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) UpdateFile(remotePath string, data []byte, args ...interface{}) (err error) {
|
||||
collection, ttl := getCollectionAndTtlFromArgs(args...)
|
||||
params := seaweedFsManagerParams{
|
||||
remotePath: remotePath,
|
||||
collection: collection,
|
||||
ttl: ttl,
|
||||
data: data,
|
||||
}
|
||||
res := m.process(params, m.updateFile)
|
||||
return res.err
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) Exists(remotePath string, args ...interface{}) (ok bool, err error) {
|
||||
_, err = m.GetFile(remotePath, args...)
|
||||
if err == nil {
|
||||
// exists
|
||||
return true, nil
|
||||
}
|
||||
if strings.Contains(err.Error(), FilerStatusNotFoundErrorMessage) {
|
||||
// not exists
|
||||
return false, nil
|
||||
}
|
||||
return ok, trace.TraceError(err)
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetFilerUrl(url string) {
|
||||
m.filerUrl = url
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetFilerAuthKey(authKey string) {
|
||||
m.authKey = authKey
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetTimeout(timeout time.Duration) {
|
||||
m.timeout = timeout
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetWorkerNum(num int) {
|
||||
m.workerNum = num
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetRetryInterval(interval time.Duration) {
|
||||
m.retryInterval = interval
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetRetryNum(num int) {
|
||||
m.retryNum = uint64(num)
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) SetMaxQps(qps int) {
|
||||
m.maxQps = qps
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) newHandle(params seaweedFsManagerParams, fn seaweedFsManagerFn) (handle seaweedFsManagerHandle) {
|
||||
return seaweedFsManagerHandle{
|
||||
params: params,
|
||||
fn: fn,
|
||||
resChan: make(chan seaweedFsManagerResults),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) start() {
|
||||
for {
|
||||
if m.closed {
|
||||
return
|
||||
}
|
||||
handle := <-m.ch
|
||||
go func() {
|
||||
if err := backoff.Retry(func() error {
|
||||
res := handle.fn(handle.params)
|
||||
if res.err != nil {
|
||||
return res.err
|
||||
}
|
||||
handle.resChan <- res
|
||||
return nil
|
||||
}, backoff.WithMaxRetries(
|
||||
backoff.NewConstantBackOff(m.retryInterval), m.retryNum),
|
||||
); err != nil {
|
||||
handle.resChan <- m.error(err)
|
||||
}
|
||||
m.wait()
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) process(params seaweedFsManagerParams, fn seaweedFsManagerFn) (res seaweedFsManagerResults) {
|
||||
handle := m.newHandle(params, fn)
|
||||
//log.Infof("handle: %v", handle)
|
||||
m.ch <- handle
|
||||
res = <-handle.resChan
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) error(err error) (res seaweedFsManagerResults) {
|
||||
if err != nil {
|
||||
trace.PrintError(err)
|
||||
}
|
||||
return seaweedFsManagerResults{err: err}
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) wait() {
|
||||
ms := float32(1) / float32(m.maxQps) * 1e3
|
||||
d := time.Duration(ms) * time.Millisecond
|
||||
time.Sleep(d)
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) getCollectionAndTtlArgsFromParams(params seaweedFsManagerParams) (args []interface{}) {
|
||||
if params.collection != "" {
|
||||
args = append(args, params.collection)
|
||||
}
|
||||
if params.ttl != "" {
|
||||
args = append(args, params.ttl)
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) listDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
var err error
|
||||
if params.isRecursive {
|
||||
res.files, err = m.ListDirRecursive(params.remotePath)
|
||||
} else {
|
||||
res.files, err = m.f.ListDir(params.remotePath)
|
||||
}
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) listDirRecursive(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
entries, err := m.f.ListDir(params.remotePath)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
for _, file := range entries {
|
||||
file = goseaweedfs.GetFileWithExtendedFields(file)
|
||||
if file.IsDir {
|
||||
file.Children, err = m.ListDirRecursive(file.FullPath)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
res.files = append(res.files, file)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) uploadFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
r, err := m.f.UploadFile(params.localPath, params.remotePath, params.collection, params.ttl)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
if r.Error != "" {
|
||||
err = errors.New(r.Error)
|
||||
return m.error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) uploadDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
args := m.getCollectionAndTtlArgsFromParams(params)
|
||||
if strings.HasSuffix(params.localPath, "/") {
|
||||
params.localPath = params.localPath[:(len(params.localPath) - 1)]
|
||||
}
|
||||
if !strings.HasPrefix(params.remotePath, "/") {
|
||||
params.remotePath = "/" + params.remotePath
|
||||
}
|
||||
files, err := goseaweedfs.ListFilesRecursive(params.localPath)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
for _, info := range files {
|
||||
newFilePath := params.remotePath + strings.Replace(info.Path, params.localPath, "", -1)
|
||||
if err := m.UploadFile(info.Path, newFilePath, args...); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) downloadFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
err := m.f.Download(params.remotePath, params.urlValues, func(reader io.Reader) error {
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
dirPath := filepath.Dir(params.localPath)
|
||||
_, err = os.Stat(dirPath)
|
||||
if err != nil {
|
||||
// if not exists, create a new directory
|
||||
if err := os.MkdirAll(dirPath, DefaultDirMode); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
}
|
||||
fileMode := DefaultFileMode
|
||||
fileInfo, err := os.Stat(params.localPath)
|
||||
if err == nil {
|
||||
// if file already exists, save file mode and remove it
|
||||
fileMode = fileInfo.Mode()
|
||||
if err := os.Remove(params.localPath); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
}
|
||||
if err := ioutil.WriteFile(params.localPath, data, fileMode); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) downloadDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
args := m.getCollectionAndTtlArgsFromParams(params)
|
||||
var files []goseaweedfs.FilerFileInfo
|
||||
files, res.err = m.ListDir(params.remotePath, true)
|
||||
for _, file := range files {
|
||||
if file.IsDir {
|
||||
if err := m.DownloadDir(file.FullPath, path.Join(params.localPath, file.Name), args...); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
} else {
|
||||
if err := m.DownloadFile(file.FullPath, path.Join(params.localPath, file.Name), args...); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) deleteFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
if err := m.f.DeleteFile(params.remotePath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) deleteDir(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
if err := m.f.DeleteDir(params.remotePath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) syncLocalToRemote(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
// args
|
||||
args := m.getCollectionAndTtlArgsFromParams(params)
|
||||
|
||||
// raise error if local path does not exist
|
||||
if _, err := os.Stat(params.localPath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
|
||||
// get files and maps
|
||||
localFiles, remoteFiles, localFilesMap, remoteFilesMap, err := getFilesAndFilesMaps(m.f, params.localPath, params.remotePath)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
|
||||
// compare remote files with local files and delete files absent in local files
|
||||
for _, remoteFile := range remoteFiles {
|
||||
// attempt to get corresponding local file
|
||||
_, ok := localFilesMap[remoteFile.FullPath]
|
||||
|
||||
if !ok {
|
||||
// file does not exist on local, delete
|
||||
if remoteFile.IsDir {
|
||||
if err := m.DeleteDir(remoteFile.FullPath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
} else {
|
||||
if err := m.DeleteFile(remoteFile.FullPath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compare local files with remote files and upload files with difference
|
||||
for _, localFile := range localFiles {
|
||||
// skip .git
|
||||
if IsGitFile(localFile) {
|
||||
continue
|
||||
}
|
||||
|
||||
// corresponding remote file path
|
||||
fileRemotePath := fmt.Sprintf("%s%s", params.remotePath, strings.Replace(localFile.Path, params.localPath, "", -1))
|
||||
|
||||
// attempt to get corresponding remote file
|
||||
remoteFile, ok := remoteFilesMap[fileRemotePath]
|
||||
|
||||
if !ok {
|
||||
// file does not exist on remote, upload
|
||||
if err := m.UploadFile(localFile.Path, fileRemotePath, args...); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
} else {
|
||||
// file exists on remote, upload if md5sum values are different
|
||||
if remoteFile.Md5 != localFile.Md5 {
|
||||
if err := m.UploadFile(localFile.Path, fileRemotePath, args...); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) syncRemoteToLocal(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
// args
|
||||
args := m.getCollectionAndTtlArgsFromParams(params)
|
||||
|
||||
// create directory if local path does not exist
|
||||
if _, err := os.Stat(params.localPath); err != nil {
|
||||
if err := os.MkdirAll(params.localPath, os.ModePerm); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// get files and maps
|
||||
localFiles, remoteFiles, localFilesMap, remoteFilesMap, err := getFilesAndFilesMaps(m.f, params.localPath, params.remotePath)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
|
||||
// compare local files with remote files and delete files absent on remote
|
||||
for _, localFile := range localFiles {
|
||||
// skip .git
|
||||
if IsGitFile(localFile) {
|
||||
continue
|
||||
}
|
||||
|
||||
// corresponding remote file path
|
||||
fileRemotePath := fmt.Sprintf("%s%s", params.remotePath, strings.Replace(localFile.Path, params.localPath, "", -1))
|
||||
|
||||
// attempt to get corresponding remote file
|
||||
_, ok := remoteFilesMap[fileRemotePath]
|
||||
|
||||
if !ok {
|
||||
// file does not exist on remote, upload
|
||||
if err := os.Remove(localFile.Path); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compare remote files with local files and download if files with difference
|
||||
for _, remoteFile := range remoteFiles {
|
||||
// directory
|
||||
if remoteFile.IsDir {
|
||||
localDirRelativePath := strings.Replace(remoteFile.FullPath, params.remotePath, "", 1)
|
||||
localDirPath := fmt.Sprintf("%s%s", params.localPath, localDirRelativePath)
|
||||
if err := m.SyncRemoteToLocal(remoteFile.FullPath, localDirPath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// local file path
|
||||
localFileRelativePath := strings.Replace(remoteFile.FullPath, params.remotePath, "", 1)
|
||||
localFilePath := fmt.Sprintf("%s%s", params.localPath, localFileRelativePath)
|
||||
|
||||
// attempt to get corresponding local file
|
||||
localFile, ok := localFilesMap[remoteFile.FullPath]
|
||||
|
||||
if !ok {
|
||||
// file does not exist on local, download
|
||||
if err := m.DownloadFile(remoteFile.FullPath, localFilePath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
} else {
|
||||
// file exists on remote, download if md5sum values are different
|
||||
if remoteFile.Md5 != localFile.Md5 {
|
||||
if err := m.DownloadFile(remoteFile.FullPath, localFilePath, args...); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) getFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
var buf bytes.Buffer
|
||||
res.err = m.f.Download(params.remotePath, params.urlValues, func(reader io.Reader) error {
|
||||
_, err := io.Copy(&buf, reader)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
res.data = buf.Bytes()
|
||||
return
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) getFileInfo(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
arr := strings.Split(params.remotePath, "/")
|
||||
dirName := strings.Join(arr[:(len(arr)-1)], "/")
|
||||
files, err := m.f.ListDir(dirName)
|
||||
if err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
for _, f := range files {
|
||||
if f.FullPath == params.remotePath {
|
||||
res.file = &f
|
||||
return
|
||||
}
|
||||
}
|
||||
return m.error(ErrorFsNotExists)
|
||||
}
|
||||
|
||||
func (m *SeaweedFsManager) updateFile(params seaweedFsManagerParams) (res seaweedFsManagerResults) {
|
||||
tmpRootDir := os.TempDir()
|
||||
tmpDirPath := path.Join(tmpRootDir, ".seaweedfs")
|
||||
if _, err := os.Stat(tmpDirPath); err != nil {
|
||||
if err := os.MkdirAll(tmpDirPath, os.ModePerm); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
tmpFilePath := path.Join(tmpDirPath, fmt.Sprintf(".%s", uuid.New().String()))
|
||||
if _, err := os.Stat(tmpFilePath); err == nil {
|
||||
if err := os.Remove(tmpFilePath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
}
|
||||
if err := ioutil.WriteFile(tmpFilePath, params.data, os.ModePerm); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
params2 := seaweedFsManagerParams{
|
||||
localPath: tmpFilePath,
|
||||
remotePath: params.remotePath,
|
||||
collection: params.collection,
|
||||
ttl: params.ttl,
|
||||
}
|
||||
if res := m.uploadFile(params2); res.err != nil {
|
||||
return m.error(res.err)
|
||||
}
|
||||
if err := os.Remove(tmpFilePath); err != nil {
|
||||
return m.error(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func NewSeaweedFsManager(opts ...Option) (m2 Manager, err error) {
|
||||
// manager
|
||||
m := &SeaweedFsManager{
|
||||
filerUrl: "http://localhost:8888",
|
||||
timeout: 5 * time.Minute,
|
||||
workerNum: 1,
|
||||
retryInterval: 500 * time.Millisecond,
|
||||
retryNum: 3,
|
||||
maxQps: 5,
|
||||
q: linkedlistqueue.New(),
|
||||
}
|
||||
|
||||
// apply options
|
||||
for _, opt := range opts {
|
||||
opt(m)
|
||||
}
|
||||
|
||||
// initialize
|
||||
if err := m.Init(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
}
|
||||
|
||||
var _seaweedFsManager Manager
|
||||
|
||||
func GetSeaweedFsManager(opts ...Option) (m2 Manager, err error) {
|
||||
if _seaweedFsManager == nil {
|
||||
_seaweedFsManager, err = NewSeaweedFsManager(opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return _seaweedFsManager, nil
|
||||
}
|
||||
56
fs/test/base.go
Normal file
56
fs/test/base.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
fs "github.com/crawlab-team/crawlab/fs"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
T, err = NewTest()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
var T *Test
|
||||
|
||||
type Test struct {
|
||||
m fs.Manager
|
||||
}
|
||||
|
||||
func (t *Test) Setup(t2 *testing.T) {
|
||||
t.Cleanup()
|
||||
t2.Cleanup(t.Cleanup)
|
||||
}
|
||||
|
||||
func (t *Test) Cleanup() {
|
||||
_ = T.m.DeleteDir("/test")
|
||||
|
||||
// wait to avoid caching
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
}
|
||||
|
||||
func NewTest() (res *Test, err error) {
|
||||
// test
|
||||
t := &Test{}
|
||||
|
||||
// filer url
|
||||
filerUrl := os.Getenv("CRAWLAB_FILER_URL")
|
||||
if filerUrl == "" {
|
||||
filerUrl = "http://localhost:8888"
|
||||
}
|
||||
|
||||
// manager
|
||||
t.m, err = fs.NewSeaweedFsManager(
|
||||
fs.WithFilerUrl(filerUrl),
|
||||
fs.WithTimeout(10*time.Second),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return t, nil
|
||||
}
|
||||
269
fs/test/bindata.go
Normal file
269
fs/test/bindata.go
Normal file
@@ -0,0 +1,269 @@
|
||||
// Code generated for package test by go-bindata DO NOT EDIT. (@generated)
|
||||
// sources:
|
||||
// bin/start.sh
|
||||
// bin/stop.sh
|
||||
package test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func bindataRead(data []byte, name string) ([]byte, error) {
|
||||
gz, err := gzip.NewReader(bytes.NewBuffer(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Read %q: %v", name, err)
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
_, err = io.Copy(&buf, gz)
|
||||
clErr := gz.Close()
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Read %q: %v", name, err)
|
||||
}
|
||||
if clErr != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
type asset struct {
|
||||
bytes []byte
|
||||
info os.FileInfo
|
||||
}
|
||||
|
||||
type bindataFileInfo struct {
|
||||
name string
|
||||
size int64
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
}
|
||||
|
||||
// Name return file name
|
||||
func (fi bindataFileInfo) Name() string {
|
||||
return fi.name
|
||||
}
|
||||
|
||||
// Size return file size
|
||||
func (fi bindataFileInfo) Size() int64 {
|
||||
return fi.size
|
||||
}
|
||||
|
||||
// Mode return file mode
|
||||
func (fi bindataFileInfo) Mode() os.FileMode {
|
||||
return fi.mode
|
||||
}
|
||||
|
||||
// Mode return file modify time
|
||||
func (fi bindataFileInfo) ModTime() time.Time {
|
||||
return fi.modTime
|
||||
}
|
||||
|
||||
// IsDir return file whether a directory
|
||||
func (fi bindataFileInfo) IsDir() bool {
|
||||
return fi.mode&os.ModeDir != 0
|
||||
}
|
||||
|
||||
// Sys return file is sys mode
|
||||
func (fi bindataFileInfo) Sys() interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _binStartSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x8f\x4b\x0e\xc2\x30\x0c\x44\xf7\x3e\xc5\x20\xd6\x4d\x58\xc3\x51\x80\x45\x4b\x1c\xd5\x22\x69\xab\x38\x2d\x3d\x3e\x6a\xa0\x7c\xc4\x05\x90\x57\x7e\xa3\x19\x8f\xb7\x1b\xdb\x48\x67\xb5\x25\xf1\x38\xa2\x62\x18\x9b\xe3\x80\xf3\x01\xb9\xe5\x8e\x80\x3d\x71\x50\x26\x20\x5e\x9d\xa4\x87\x4c\x5e\xe8\x69\x98\x61\x47\x4d\x36\xf4\x97\x3a\x94\xa8\x1b\xb3\xfb\xb0\x97\x55\x39\x4d\x9c\x70\x22\x00\xa8\x5e\x31\x2b\x88\xb5\x66\x4e\xe6\x87\x4f\x7d\x18\x23\x2f\xdc\x88\x9b\xbf\x35\x19\x50\x6e\xb6\xbd\xe6\x37\x33\x8d\x74\x0e\x3b\x53\x66\xc5\x5e\x02\xa7\xf5\x0b\x63\x95\xeb\xa5\x94\x57\xfb\x37\xdd\xbc\xd0\x3d\x00\x00\xff\xff\x09\xef\x85\x28\x89\x01\x00\x00")
|
||||
|
||||
func binStartShBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
_binStartSh,
|
||||
"bin/start.sh",
|
||||
)
|
||||
}
|
||||
|
||||
func binStartSh() (*asset, error) {
|
||||
bytes, err := binStartShBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "bin/start.sh", size: 393, mode: os.FileMode(493), modTime: time.Unix(1621153670, 0)}
|
||||
a := &asset{bytes: bytes, info: info}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
var _binStopSh = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x52\x56\xd4\x4f\xca\xcc\xd3\x2f\xce\xe0\xe2\xca\xce\xcc\xc9\x51\xd0\xb5\x54\x48\x28\x28\x56\x48\xac\x28\xad\x49\x2f\x4a\x2d\x50\x28\x4f\x4d\x4d\x81\xb0\x74\xcb\x14\x40\x74\x4d\x62\x79\xb6\x82\x7a\x75\x41\x51\x66\x5e\x89\x42\x8c\x8a\x51\xad\x7a\x4d\x45\x62\x51\x7a\x71\x02\x20\x00\x00\xff\xff\x6b\x5f\x02\x48\x4a\x00\x00\x00")
|
||||
|
||||
func binStopShBytes() ([]byte, error) {
|
||||
return bindataRead(
|
||||
_binStopSh,
|
||||
"bin/stop.sh",
|
||||
)
|
||||
}
|
||||
|
||||
func binStopSh() (*asset, error) {
|
||||
bytes, err := binStopShBytes()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
info := bindataFileInfo{name: "bin/stop.sh", size: 74, mode: os.FileMode(493), modTime: time.Unix(1621154552, 0)}
|
||||
a := &asset{bytes: bytes, info: info}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Asset loads and returns the asset for the given name.
|
||||
// It returns an error if the asset could not be found or
|
||||
// could not be loaded.
|
||||
func Asset(name string) ([]byte, error) {
|
||||
cannonicalName := strings.Replace(name, "\\", "/", -1)
|
||||
if f, ok := _bindata[cannonicalName]; ok {
|
||||
a, err := f()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
|
||||
}
|
||||
return a.bytes, nil
|
||||
}
|
||||
return nil, fmt.Errorf("Asset %s not found", name)
|
||||
}
|
||||
|
||||
// MustAsset is like Asset but panics when Asset would return an error.
|
||||
// It simplifies safe initialization of global variables.
|
||||
func MustAsset(name string) []byte {
|
||||
a, err := Asset(name)
|
||||
if err != nil {
|
||||
panic("asset: Asset(" + name + "): " + err.Error())
|
||||
}
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
// AssetInfo loads and returns the asset info for the given name.
|
||||
// It returns an error if the asset could not be found or
|
||||
// could not be loaded.
|
||||
func AssetInfo(name string) (os.FileInfo, error) {
|
||||
cannonicalName := strings.Replace(name, "\\", "/", -1)
|
||||
if f, ok := _bindata[cannonicalName]; ok {
|
||||
a, err := f()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
|
||||
}
|
||||
return a.info, nil
|
||||
}
|
||||
return nil, fmt.Errorf("AssetInfo %s not found", name)
|
||||
}
|
||||
|
||||
// AssetNames returns the names of the assets.
|
||||
func AssetNames() []string {
|
||||
names := make([]string, 0, len(_bindata))
|
||||
for name := range _bindata {
|
||||
names = append(names, name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// _bindata is a table, holding each asset generator, mapped to its name.
|
||||
var _bindata = map[string]func() (*asset, error){
|
||||
"bin/start.sh": binStartSh,
|
||||
"bin/stop.sh": binStopSh,
|
||||
}
|
||||
|
||||
// AssetDir returns the file names below a certain
|
||||
// directory embedded in the file by go-bindata.
|
||||
// For example if you run go-bindata on data/... and data contains the
|
||||
// following hierarchy:
|
||||
// data/
|
||||
// foo.txt
|
||||
// img/
|
||||
// a.png
|
||||
// b.png
|
||||
// then AssetDir("data") would return []string{"foo.txt", "img"}
|
||||
// AssetDir("data/img") would return []string{"a.png", "b.png"}
|
||||
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
|
||||
// AssetDir("") will return []string{"data"}.
|
||||
func AssetDir(name string) ([]string, error) {
|
||||
node := _bintree
|
||||
if len(name) != 0 {
|
||||
cannonicalName := strings.Replace(name, "\\", "/", -1)
|
||||
pathList := strings.Split(cannonicalName, "/")
|
||||
for _, p := range pathList {
|
||||
node = node.Children[p]
|
||||
if node == nil {
|
||||
return nil, fmt.Errorf("Asset %s not found", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if node.Func != nil {
|
||||
return nil, fmt.Errorf("Asset %s not found", name)
|
||||
}
|
||||
rv := make([]string, 0, len(node.Children))
|
||||
for childName := range node.Children {
|
||||
rv = append(rv, childName)
|
||||
}
|
||||
return rv, nil
|
||||
}
|
||||
|
||||
type bintree struct {
|
||||
Func func() (*asset, error)
|
||||
Children map[string]*bintree
|
||||
}
|
||||
|
||||
var _bintree = &bintree{nil, map[string]*bintree{
|
||||
"bin": &bintree{nil, map[string]*bintree{
|
||||
"start.sh": &bintree{binStartSh, map[string]*bintree{}},
|
||||
"stop.sh": &bintree{binStopSh, map[string]*bintree{}},
|
||||
}},
|
||||
}}
|
||||
|
||||
// RestoreAsset restores an asset under the given directory
|
||||
func RestoreAsset(dir, name string) error {
|
||||
data, err := Asset(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info, err := AssetInfo(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RestoreAssets restores an asset under the given directory recursively
|
||||
func RestoreAssets(dir, name string) error {
|
||||
children, err := AssetDir(name)
|
||||
// File
|
||||
if err != nil {
|
||||
return RestoreAsset(dir, name)
|
||||
}
|
||||
// Dir
|
||||
for _, child := range children {
|
||||
err = RestoreAssets(dir, filepath.Join(name, child))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func _filePath(dir, name string) string {
|
||||
cannonicalName := strings.Replace(name, "\\", "/", -1)
|
||||
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
|
||||
}
|
||||
21
fs/test/main_test.go
Normal file
21
fs/test/main_test.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
// before test
|
||||
//if err := StartTestSeaweedFs(); err != nil {
|
||||
// panic(err)
|
||||
//}
|
||||
|
||||
// test
|
||||
m.Run()
|
||||
|
||||
// close
|
||||
_ = T.m.Close()
|
||||
|
||||
// after test
|
||||
//_ = StopTestSeaweedFs()
|
||||
}
|
||||
473
fs/test/seaweedfs_test.go
Normal file
473
fs/test/seaweedfs_test.go
Normal file
@@ -0,0 +1,473 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/apex/log"
|
||||
"github.com/crawlab-team/crawlab/fs"
|
||||
"github.com/crawlab-team/crawlab/fs/lib/copy"
|
||||
"github.com/stretchr/testify/require"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewSeaweedFsManager(t *testing.T) {
|
||||
_, err := fs.NewSeaweedFsManager()
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_ListDir(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data/nested", "/test/data/nested")
|
||||
require.Nil(t, err)
|
||||
|
||||
valid := false
|
||||
files, err := T.m.ListDir("/test/data", true)
|
||||
require.Nil(t, err)
|
||||
for _, f1 := range files {
|
||||
if f1.Name == "nested" && f1.Children != nil {
|
||||
for _, f2 := range f1.Children {
|
||||
if f2.Name == "nested_test_data.txt" {
|
||||
valid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, valid)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_UploadFile(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
files, err := T.m.ListDir("/test/data", true)
|
||||
require.Nil(t, err)
|
||||
valid := false
|
||||
for _, file := range files {
|
||||
if file.Name == "test_data.txt" {
|
||||
valid = true
|
||||
}
|
||||
}
|
||||
require.True(t, valid)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_UploadDir(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data/nested", "/test/data/nested")
|
||||
require.Nil(t, err)
|
||||
|
||||
valid := false
|
||||
files, err := T.m.ListDir("/test/data", true)
|
||||
require.Nil(t, err)
|
||||
for _, f1 := range files {
|
||||
if f1.Name == "nested" && f1.Children != nil {
|
||||
for _, f2 := range f1.Children {
|
||||
if f2.Name == "nested_test_data.txt" {
|
||||
valid = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
require.True(t, valid)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_GetFile(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err := T.m.GetFile("/test/data/test_data.txt")
|
||||
require.Equal(t, "this is a test data", string(data))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_DownloadFile(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.DownloadFile("/test/data/test_data.txt", "./tmp/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err := ioutil.ReadFile("./tmp/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.NotEmpty(t, data)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_DownloadDir(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data/nested", "/test/data/nested")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.DownloadDir("/test/data", "./tmp/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err := ioutil.ReadFile("./data/nested/nested_test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.NotEmpty(t, data)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_DeleteFile(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadFile("./data/test_data.txt", "/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.DeleteFile("/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
files, err := T.m.ListDir("/test/data", true)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, 0, len(files))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_DeleteDir(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.DeleteDir("/test/data/nested")
|
||||
require.Nil(t, err)
|
||||
|
||||
files, err := T.m.ListDir("/test/data", true)
|
||||
require.Nil(t, err)
|
||||
valid := true
|
||||
for _, file := range files {
|
||||
if file.Name == "nested" && file.IsDir {
|
||||
valid = false
|
||||
}
|
||||
}
|
||||
require.True(t, valid)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_SyncLocalToRemote(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = copy.CopyDirectory("./data", "./tmp/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.SyncLocalToRemote("./tmp/data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err := T.m.GetFile("/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, "this is a test data", string(data))
|
||||
|
||||
data, err = T.m.GetFile("/test/data/nested/nested_test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, "this is nested test data", string(data))
|
||||
|
||||
err = ioutil.WriteFile("./tmp/data/test_data.txt", []byte("this is changed data"), os.ModePerm)
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.SyncLocalToRemote("./tmp/data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err = T.m.GetFile("/test/data/test_data.txt")
|
||||
require.Equal(t, "this is changed data", string(data))
|
||||
|
||||
err = os.Remove("./tmp/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.SyncLocalToRemote("./tmp/data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
valid := true
|
||||
files, err := T.m.ListDir("/test/data", true)
|
||||
for _, file := range files {
|
||||
if file.Name == "test_data.txt" {
|
||||
valid = false
|
||||
}
|
||||
}
|
||||
require.True(t, valid)
|
||||
|
||||
// check if directory is deleted after sync
|
||||
err = ioutil.WriteFile("./tmp/test.txt", []byte("test"), os.ModePerm)
|
||||
require.Nil(t, err)
|
||||
err = T.m.UploadFile("./tmp/test.txt", "/test/data/folder1/test.txt")
|
||||
require.Nil(t, err)
|
||||
time.Sleep(1 * time.Second)
|
||||
err = T.m.SyncLocalToRemote("./tmp/data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
valid = true
|
||||
files, err = T.m.ListDir("/test/data", true)
|
||||
for _, file := range files {
|
||||
if strings.Contains(file.FullPath, "folder1") {
|
||||
valid = false
|
||||
}
|
||||
}
|
||||
require.True(t, valid)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_SyncRemoteToLocal(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
if _, err := os.Stat("./tmp/data"); err == nil {
|
||||
err = os.RemoveAll("./tmp/data")
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
err = T.m.UploadDir("./data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.SyncRemoteToLocal("/test/data", "./tmp/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err := ioutil.ReadFile("./tmp/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, "this is a test data", string(data))
|
||||
|
||||
data, err = ioutil.ReadFile("./tmp/data/nested/nested_test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, "this is nested test data", string(data))
|
||||
|
||||
err = T.m.UpdateFile("/test/data/test_data.txt", []byte("this is changed data"))
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.SyncRemoteToLocal("/test/data", "./tmp/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err = ioutil.ReadFile("./tmp/data/test_data.txt")
|
||||
require.Equal(t, "this is changed data", string(data))
|
||||
|
||||
err = T.m.DeleteFile("/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.SyncRemoteToLocal("/test/data", "./tmp/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
_, err = os.Stat("./tmp/data/test_data.txt")
|
||||
require.NotNil(t, err)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_UpdateFile(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
err = T.m.UpdateFile("/test/data/test_data.txt", []byte("this is changed data"))
|
||||
require.Nil(t, err)
|
||||
|
||||
data, err := T.m.GetFile("/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, "this is changed data", string(data))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_Exists(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
ok, err := T.m.Exists("/test/data/test_data.txt")
|
||||
require.Nil(t, err)
|
||||
require.True(t, ok)
|
||||
|
||||
ok, err = T.m.Exists("/test/data/test_data_404.txt")
|
||||
require.Nil(t, err)
|
||||
require.False(t, ok)
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_ListDirPressure(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
err = T.m.UploadDir("./data/nested", "/test/data/nested")
|
||||
require.Nil(t, err)
|
||||
|
||||
n := int(1e3)
|
||||
doneNum := 0
|
||||
errNum := 0
|
||||
startTs := time.Now()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
_, err := T.m.ListDir("test/data", true)
|
||||
wg.Done()
|
||||
if err != nil {
|
||||
errNum++
|
||||
}
|
||||
doneNum++
|
||||
log.Infof("list dir: %d/%d", doneNum, n)
|
||||
require.Nil(t, err)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
endTs := time.Now()
|
||||
duration := endTs.Sub(startTs).Milliseconds()
|
||||
|
||||
fmt.Println(fmt.Sprintf("total: %d", n))
|
||||
fmt.Println(fmt.Sprintf("errors: %d", errNum))
|
||||
fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n)))
|
||||
fmt.Println(fmt.Sprintf("duration: %dms", duration))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_UploadFilePressure(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
n := int(1e3)
|
||||
doneNum := 0
|
||||
errNum := 0
|
||||
startTs := time.Now()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
err = T.m.UploadFile("./data/test_data.txt", fmt.Sprintf("/test/data/test_data_%d.txt", i))
|
||||
wg.Done()
|
||||
if err != nil {
|
||||
errNum++
|
||||
}
|
||||
doneNum++
|
||||
log.Infof("upload file: %d/%d", doneNum, n)
|
||||
require.Nil(t, err)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
endTs := time.Now()
|
||||
duration := endTs.Sub(startTs).Milliseconds()
|
||||
|
||||
fmt.Println(fmt.Sprintf("total: %d", n))
|
||||
fmt.Println(fmt.Sprintf("errors: %d", errNum))
|
||||
fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n)))
|
||||
fmt.Println(fmt.Sprintf("duration: %dms", duration))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_UploadDirPressure(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
n := int(1e3)
|
||||
doneNum := 0
|
||||
errNum := 0
|
||||
startTs := time.Now()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
err = T.m.UploadDir("./data/nested", "/test/data")
|
||||
wg.Done()
|
||||
if err != nil {
|
||||
errNum++
|
||||
}
|
||||
doneNum++
|
||||
log.Infof("upload dir: %d/%d", doneNum, n)
|
||||
require.Nil(t, err)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
endTs := time.Now()
|
||||
duration := endTs.Sub(startTs).Milliseconds()
|
||||
|
||||
fmt.Println(fmt.Sprintf("total: %d", n))
|
||||
fmt.Println(fmt.Sprintf("errors: %d", errNum))
|
||||
fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n)))
|
||||
fmt.Println(fmt.Sprintf("duration: %dms", duration))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_SyncRemoteToLocalPressure(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
if _, err := os.Stat("./tmp/data"); err == nil {
|
||||
err = os.RemoveAll("./tmp/data")
|
||||
require.Nil(t, err)
|
||||
}
|
||||
|
||||
err = T.m.UploadDir("./data", "/test/data")
|
||||
require.Nil(t, err)
|
||||
|
||||
n := int(1e3)
|
||||
doneNum := 0
|
||||
errNum := 0
|
||||
startTs := time.Now()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
err = T.m.SyncRemoteToLocal("/test/data", fmt.Sprintf("./tmp/data_%d", i))
|
||||
wg.Done()
|
||||
if err != nil {
|
||||
errNum++
|
||||
}
|
||||
doneNum++
|
||||
log.Infof("updated: %d/%d", doneNum, n)
|
||||
require.Nil(t, err)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
endTs := time.Now()
|
||||
duration := endTs.Sub(startTs).Milliseconds()
|
||||
|
||||
fmt.Println(fmt.Sprintf("total: %d", n))
|
||||
fmt.Println(fmt.Sprintf("errors: %d", errNum))
|
||||
fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n)))
|
||||
fmt.Println(fmt.Sprintf("duration: %dms", duration))
|
||||
}
|
||||
|
||||
func TestSeaweedFsManager_UpdateFilePressure(t *testing.T) {
|
||||
var err error
|
||||
T.Setup(t)
|
||||
|
||||
n := int(5e3)
|
||||
doneNum := 0
|
||||
errNum := 0
|
||||
startTs := time.Now()
|
||||
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
err = T.m.UpdateFile(fmt.Sprintf("/test/data/test_data_%5d.txt", i), []byte(fmt.Sprintf("this is test data: %5d", i)))
|
||||
wg.Done()
|
||||
if err != nil {
|
||||
errNum++
|
||||
}
|
||||
doneNum++
|
||||
log.Infof("updated: %d/%d", doneNum, n)
|
||||
require.Nil(t, err)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
endTs := time.Now()
|
||||
duration := endTs.Sub(startTs).Milliseconds()
|
||||
|
||||
fmt.Println(fmt.Sprintf("total: %d", n))
|
||||
fmt.Println(fmt.Sprintf("errors: %d", errNum))
|
||||
fmt.Println(fmt.Sprintf("error rate: %.3f", float32(errNum)/float32(n)))
|
||||
fmt.Println(fmt.Sprintf("duration: %dms", duration))
|
||||
}
|
||||
124
fs/test/utils.go
Normal file
124
fs/test/utils.go
Normal file
@@ -0,0 +1,124 @@
|
||||
package test
|
||||
|
||||
import (
|
||||
"github.com/apex/log"
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"github.com/crawlab-team/crawlab/trace"
|
||||
"github.com/google/uuid"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
TmpDir, err = filepath.Abs("tmp")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := os.Stat(TmpDir); err != nil {
|
||||
if err := os.MkdirAll(TmpDir, os.ModePerm); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
//TmpDir = getTmpDir()
|
||||
}
|
||||
|
||||
var TmpDir string
|
||||
|
||||
func StartTestSeaweedFs() (err error) {
|
||||
// skip if CRAWLAB_IGNORE_WEED is set true
|
||||
if os.Getenv("CRAWLAB_IGNORE_WEED") != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// write to start.sh and stop.sh
|
||||
if err := writeShFiles(TmpDir); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
// run weed
|
||||
go runCmd(exec.Command("sh", "./start.sh"), TmpDir)
|
||||
|
||||
// wait for containers to be ready
|
||||
time.Sleep(5 * time.Second)
|
||||
f := func() error {
|
||||
_, err := T.m.ListDir("/", true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
b := backoff.WithMaxRetries(backoff.NewConstantBackOff(5*time.Second), 5)
|
||||
nt := func(err error, duration time.Duration) {
|
||||
log.Infof("seaweedfs services not ready, re-attempt in %.1f seconds", duration.Seconds())
|
||||
}
|
||||
err = backoff.RetryNotify(f, b, nt)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func StopTestSeaweedFs() (err error) {
|
||||
// skip if CRAWLAB_IGNORE_WEED is set true
|
||||
if os.Getenv("CRAWLAB_IGNORE_WEED") != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// stop seaweedfs
|
||||
if err := runCmd(exec.Command("sh", "./stop.sh"), TmpDir); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
// remove tmp folder
|
||||
if err := os.RemoveAll(TmpDir); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeShFiles(dirPath string) (err error) {
|
||||
fileNames := []string{
|
||||
"start.sh",
|
||||
"stop.sh",
|
||||
}
|
||||
|
||||
for _, fileName := range fileNames {
|
||||
data, err := Asset("bin/" + fileName)
|
||||
if err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
filePath := path.Join(dirPath, fileName)
|
||||
if err := ioutil.WriteFile(filePath, data, os.FileMode(0766)); err != nil {
|
||||
return trace.TraceError(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func runCmd(cmd *exec.Cmd, dirPath string) (err error) {
|
||||
log.Infof("running cmd: %v", cmd)
|
||||
cmd.Dir = dirPath
|
||||
//cmd.Stdout = os.Stdout
|
||||
//cmd.Stderr = os.Stdout
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func getTmpDir() string {
|
||||
id, _ := uuid.NewUUID()
|
||||
tmpDir := path.Join(os.TempDir(), id.String())
|
||||
if _, err := os.Stat(tmpDir); err != nil {
|
||||
if err := os.MkdirAll(tmpDir, os.FileMode(0766)); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
return tmpDir
|
||||
}
|
||||
89
fs/utils.go
Normal file
89
fs/utils.go
Normal file
@@ -0,0 +1,89 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/crawlab-team/goseaweedfs"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func IsGitFile(file goseaweedfs.FileInfo) (res bool) {
|
||||
// skip .git
|
||||
res, err := regexp.MatchString("/?\\.git/", file.Path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func getCollectionAndTtlFromArgs(args ...interface{}) (collection, ttl string) {
|
||||
if len(args) > 0 {
|
||||
collection = args[0].(string)
|
||||
}
|
||||
if len(args) > 1 {
|
||||
ttl = args[1].(string)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func getUrlValuesFromArgs(args ...interface{}) (values url.Values) {
|
||||
if len(args) > 0 {
|
||||
values = args[0].(url.Values)
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
func getFilesAndFilesMaps(f *goseaweedfs.Filer, localPath, remotePath string) (localFiles []goseaweedfs.FileInfo, remoteFiles []goseaweedfs.FilerFileInfo, localFilesMap map[string]goseaweedfs.FileInfo, remoteFilesMap map[string]goseaweedfs.FilerFileInfo, err error) {
|
||||
// declare maps
|
||||
localFilesMap = map[string]goseaweedfs.FileInfo{}
|
||||
remoteFilesMap = map[string]goseaweedfs.FilerFileInfo{}
|
||||
|
||||
// cache local files info
|
||||
localFiles, err = goseaweedfs.ListLocalFilesRecursive(localPath)
|
||||
if err != nil {
|
||||
return localFiles, remoteFiles, localFilesMap, remoteFilesMap, err
|
||||
}
|
||||
for _, file := range localFiles {
|
||||
fileRemotePath := fmt.Sprintf("%s%s", remotePath, strings.Replace(file.Path, localPath, "", -1))
|
||||
localFilesMap[fileRemotePath] = file
|
||||
|
||||
// directory
|
||||
dirRemotePath := filepath.Dir(fileRemotePath)
|
||||
_, ok := localFilesMap[dirRemotePath]
|
||||
if !ok {
|
||||
localFilesMap[dirRemotePath] = goseaweedfs.FileInfo{
|
||||
Name: filepath.Base(dirRemotePath),
|
||||
Path: dirRemotePath,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cache remote files info
|
||||
remoteFiles, err = f.ListDirRecursive(remotePath)
|
||||
if err != nil {
|
||||
if err.Error() != FilerResponseNotFoundErrorMessage {
|
||||
return localFiles, remoteFiles, localFilesMap, remoteFilesMap, err
|
||||
}
|
||||
err = nil
|
||||
}
|
||||
remoteFiles = getFlattenRemoteFiles(remoteFiles)
|
||||
for _, file := range remoteFiles {
|
||||
remoteFilesMap[file.FullPath] = file
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func getFlattenRemoteFiles(files []goseaweedfs.FilerFileInfo) (flattenFiles []goseaweedfs.FilerFileInfo) {
|
||||
flattenFiles = []goseaweedfs.FilerFileInfo{}
|
||||
for _, file := range files {
|
||||
flattenFiles = append(flattenFiles, file)
|
||||
|
||||
if file.IsDir {
|
||||
flattenFiles = append(flattenFiles, getFlattenRemoteFiles(file.Children)...)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user