diff --git a/Dockerfile b/Dockerfile index 12002ab4..893cf6fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,6 @@ WORKDIR /go/src/app COPY ./backend . ENV GO111MODULE on -#ENV GOPROXY https://mirrors.aliyun.com/goproxy/ ENV GOPROXY https://goproxy.io RUN go install -v ./... @@ -15,7 +14,6 @@ ADD ./frontend /app WORKDIR /app # install frontend -#RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org RUN npm install -g yarn && yarn install RUN npm run build:prod diff --git a/Dockerfile.local b/Dockerfile.local new file mode 100644 index 00000000..c08719aa --- /dev/null +++ b/Dockerfile.local @@ -0,0 +1,60 @@ + +FROM golang:1.12 AS backend-build + +WORKDIR /go/src/app +COPY ./backend . + +ENV GO111MODULE on +ENV GOPROXY https://mirrors.aliyun.com/goproxy/ + +RUN go install -v ./... + +FROM node:8.16.0-alpine AS frontend-build + +ADD ./frontend /app +WORKDIR /app + +# install frontend +RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org + +RUN npm run build:prod + +# images +FROM ubuntu:latest + +ADD . /app + +# set as non-interactive +ENV DEBIAN_FRONTEND noninteractive + +# install packages +RUN apt-get update \ + && apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \ + && ln -s /usr/bin/pip3 /usr/local/bin/pip \ + && ln -s /usr/bin/python3 /usr/local/bin/python + +# install backend +RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple + +# copy backend files +COPY --from=backend-build /go/src/app . +COPY --from=backend-build /go/bin/crawlab /usr/local/bin + +# install nginx +RUN apt-get -y install nginx + +# copy frontend files +COPY --from=frontend-build /app/dist /app/dist +COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d + +# working directory +WORKDIR /app/backend + +# frontend port +EXPOSE 8080 + +# backend port +EXPOSE 8000 + +# start backend +CMD ["/bin/sh", "/app/docker_init.sh"] \ No newline at end of file diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go new file mode 100644 index 00000000..c01069c9 --- /dev/null +++ b/backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go @@ -0,0 +1,7 @@ +// Utility package for extracting JWT tokens from +// HTTP requests. +// +// The main function is ParseFromRequest and it's WithClaims variant. +// See examples for how to use the various Extractor implementations +// or roll your own. +package request diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go new file mode 100644 index 00000000..14414fe2 --- /dev/null +++ b/backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go @@ -0,0 +1,81 @@ +package request + +import ( + "errors" + "net/http" +) + +// Errors +var ( + ErrNoTokenInRequest = errors.New("no token present in request") +) + +// Interface for extracting a token from an HTTP request. +// The ExtractToken method should return a token string or an error. +// If no token is present, you must return ErrNoTokenInRequest. +type Extractor interface { + ExtractToken(*http.Request) (string, error) +} + +// Extractor for finding a token in a header. Looks at each specified +// header in order until there's a match +type HeaderExtractor []string + +func (e HeaderExtractor) ExtractToken(req *http.Request) (string, error) { + // loop over header names and return the first one that contains data + for _, header := range e { + if ah := req.Header.Get(header); ah != "" { + return ah, nil + } + } + return "", ErrNoTokenInRequest +} + +// Extract token from request arguments. This includes a POSTed form or +// GET URL arguments. Argument names are tried in order until there's a match. +// This extractor calls `ParseMultipartForm` on the request +type ArgumentExtractor []string + +func (e ArgumentExtractor) ExtractToken(req *http.Request) (string, error) { + // Make sure form is parsed + req.ParseMultipartForm(10e6) + + // loop over arg names and return the first one that contains data + for _, arg := range e { + if ah := req.Form.Get(arg); ah != "" { + return ah, nil + } + } + + return "", ErrNoTokenInRequest +} + +// Tries Extractors in order until one returns a token string or an error occurs +type MultiExtractor []Extractor + +func (e MultiExtractor) ExtractToken(req *http.Request) (string, error) { + // loop over header names and return the first one that contains data + for _, extractor := range e { + if tok, err := extractor.ExtractToken(req); tok != "" { + return tok, nil + } else if err != ErrNoTokenInRequest { + return "", err + } + } + return "", ErrNoTokenInRequest +} + +// Wrap an Extractor in this to post-process the value before it's handed off. +// See AuthorizationHeaderExtractor for an example +type PostExtractionFilter struct { + Extractor + Filter func(string) (string, error) +} + +func (e *PostExtractionFilter) ExtractToken(req *http.Request) (string, error) { + if tok, err := e.Extractor.ExtractToken(req); tok != "" { + return e.Filter(tok) + } else { + return "", err + } +} diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go new file mode 100644 index 00000000..5948694a --- /dev/null +++ b/backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go @@ -0,0 +1,28 @@ +package request + +import ( + "strings" +) + +// Strips 'Bearer ' prefix from bearer token string +func stripBearerPrefixFromTokenString(tok string) (string, error) { + // Should be a bearer token + if len(tok) > 6 && strings.ToUpper(tok[0:7]) == "BEARER " { + return tok[7:], nil + } + return tok, nil +} + +// Extract bearer token from Authorization header +// Uses PostExtractionFilter to strip "Bearer " prefix from header +var AuthorizationHeaderExtractor = &PostExtractionFilter{ + HeaderExtractor{"Authorization"}, + stripBearerPrefixFromTokenString, +} + +// Extractor for OAuth2 access tokens. Looks in 'Authorization' +// header then 'access_token' argument for a token. +var OAuth2Extractor = &MultiExtractor{ + AuthorizationHeaderExtractor, + ArgumentExtractor{"access_token"}, +} diff --git a/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go b/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go new file mode 100644 index 00000000..70525cfa --- /dev/null +++ b/backend/vendor/github.com/dgrijalva/jwt-go/request/request.go @@ -0,0 +1,68 @@ +package request + +import ( + "github.com/dgrijalva/jwt-go" + "net/http" +) + +// Extract and parse a JWT token from an HTTP request. +// This behaves the same as Parse, but accepts a request and an extractor +// instead of a token string. The Extractor interface allows you to define +// the logic for extracting a token. Several useful implementations are provided. +// +// You can provide options to modify parsing behavior +func ParseFromRequest(req *http.Request, extractor Extractor, keyFunc jwt.Keyfunc, options ...ParseFromRequestOption) (token *jwt.Token, err error) { + // Create basic parser struct + p := &fromRequestParser{req, extractor, nil, nil} + + // Handle options + for _, option := range options { + option(p) + } + + // Set defaults + if p.claims == nil { + p.claims = jwt.MapClaims{} + } + if p.parser == nil { + p.parser = &jwt.Parser{} + } + + // perform extract + tokenString, err := p.extractor.ExtractToken(req) + if err != nil { + return nil, err + } + + // perform parse + return p.parser.ParseWithClaims(tokenString, p.claims, keyFunc) +} + +// ParseFromRequest but with custom Claims type +// DEPRECATED: use ParseFromRequest and the WithClaims option +func ParseFromRequestWithClaims(req *http.Request, extractor Extractor, claims jwt.Claims, keyFunc jwt.Keyfunc) (token *jwt.Token, err error) { + return ParseFromRequest(req, extractor, keyFunc, WithClaims(claims)) +} + +type fromRequestParser struct { + req *http.Request + extractor Extractor + claims jwt.Claims + parser *jwt.Parser +} + +type ParseFromRequestOption func(*fromRequestParser) + +// Parse with custom claims +func WithClaims(claims jwt.Claims) ParseFromRequestOption { + return func(p *fromRequestParser) { + p.claims = claims + } +} + +// Parse using a custom parser +func WithParser(parser *jwt.Parser) ParseFromRequestOption { + return func(p *fromRequestParser) { + p.parser = parser + } +} diff --git a/examples/run_docker_master.sh b/examples/run_docker_master.sh new file mode 100644 index 00000000..531f4dda --- /dev/null +++ b/examples/run_docker_master.sh @@ -0,0 +1,10 @@ +docker run -d --rm --name crawlab \ + -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \ + -e CRAWLAB_MONGO_HOST=192.168.99.1 \ + -e CRAWLAB_SERVER_MASTER=Y \ + -e CRAWLAB_API_ADDRESS=192.168.99.100:8000 \ + -e CRAWLAB_SPIDER_PATH=/app/spiders \ + -p 8080:8080 \ + -p 8000:8000 \ + -v /var/logs/crawlab:/var/logs/crawlab \ + tikazyq/crawlab:0.3.0 \ No newline at end of file diff --git a/examples/run_docker_worker.sh b/examples/run_docker_worker.sh new file mode 100644 index 00000000..48eab1cc --- /dev/null +++ b/examples/run_docker_worker.sh @@ -0,0 +1,6 @@ +docker run --rm --name crawlab \ + -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \ + -e CRAWLAB_MONGO_HOST=192.168.99.1 \ + -e CRAWLAB_SERVER_MASTER=N \ + -v /var/logs/crawlab:/var/logs/crawlab \ + tikazyq/crawlab:0.3.0 \ No newline at end of file