Merge pull request #85 from tikazyq/develop

Develop

Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /go/src/app
 COPY ./backend .
 
 ENV GO111MODULE on
-ENV GOPROXY https://mirrors.aliyun.com/goproxy/
+ENV GOPROXY https://goproxy.io
 
 RUN go install -v ./...
 
@@ -14,7 +14,7 @@ ADD ./frontend /app
 WORKDIR /app
 
 # install frontend
-RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org
+RUN npm install -g yarn && yarn install
 
 RUN npm run build:prod
 

Dockerfile.local

@@ -0,0 +1,60 @@
+
+FROM golang:1.12 AS backend-build
+
+WORKDIR /go/src/app
+COPY ./backend .
+
+ENV GO111MODULE on
+ENV GOPROXY https://mirrors.aliyun.com/goproxy/
+
+RUN go install -v ./...
+
+FROM node:8.16.0-alpine AS frontend-build
+
+ADD ./frontend /app
+WORKDIR /app
+
+# install frontend
+RUN npm install -g yarn && yarn install --registry=https://registry.npm.taobao.org
+
+RUN npm run build:prod
+
+# images
+FROM ubuntu:latest
+
+ADD . /app
+
+# set as non-interactive
+ENV DEBIAN_FRONTEND noninteractive
+
+# install packages
+RUN apt-get update \
+	&& apt-get install -y curl git net-tools iputils-ping ntp ntpdate python3 python3-pip \
+	&& ln -s /usr/bin/pip3 /usr/local/bin/pip \
+	&& ln -s /usr/bin/python3 /usr/local/bin/python
+
+# install backend
+RUN pip install scrapy pymongo bs4 requests -i https://pypi.tuna.tsinghua.edu.cn/simple
+
+# copy backend files
+COPY --from=backend-build /go/src/app .
+COPY --from=backend-build /go/bin/crawlab /usr/local/bin
+
+# install nginx
+RUN apt-get -y install nginx
+
+# copy frontend files
+COPY --from=frontend-build /app/dist /app/dist
+COPY --from=frontend-build /app/conf/crawlab.conf /etc/nginx/conf.d
+
+# working directory
+WORKDIR /app/backend
+
+# frontend port
+EXPOSE 8080
+
+# backend port
+EXPOSE 8000
+
+# start backend
+CMD ["/bin/sh", "/app/docker_init.sh"]

backend/vendor/github.com/dgrijalva/jwt-go/request/doc.go

@@ -0,0 +1,7 @@
+// Utility package for extracting JWT tokens from
+// HTTP requests.
+//
+// The main function is ParseFromRequest and it's WithClaims variant.
+// See examples for how to use the various Extractor implementations
+// or roll your own.
+package request

backend/vendor/github.com/dgrijalva/jwt-go/request/extractor.go

@@ -0,0 +1,81 @@
+package request
+
+import (
+	"errors"
+	"net/http"
+)
+
+// Errors
+var (
+	ErrNoTokenInRequest = errors.New("no token present in request")
+)
+
+// Interface for extracting a token from an HTTP request.
+// The ExtractToken method should return a token string or an error.
+// If no token is present, you must return ErrNoTokenInRequest.
+type Extractor interface {
+	ExtractToken(*http.Request) (string, error)
+}
+
+// Extractor for finding a token in a header.  Looks at each specified
+// header in order until there's a match
+type HeaderExtractor []string
+
+func (e HeaderExtractor) ExtractToken(req *http.Request) (string, error) {
+	// loop over header names and return the first one that contains data
+	for _, header := range e {
+		if ah := req.Header.Get(header); ah != "" {
+			return ah, nil
+		}
+	}
+	return "", ErrNoTokenInRequest
+}
+
+// Extract token from request arguments.  This includes a POSTed form or
+// GET URL arguments.  Argument names are tried in order until there's a match.
+// This extractor calls `ParseMultipartForm` on the request
+type ArgumentExtractor []string
+
+func (e ArgumentExtractor) ExtractToken(req *http.Request) (string, error) {
+	// Make sure form is parsed
+	req.ParseMultipartForm(10e6)
+
+	// loop over arg names and return the first one that contains data
+	for _, arg := range e {
+		if ah := req.Form.Get(arg); ah != "" {
+			return ah, nil
+		}
+	}
+
+	return "", ErrNoTokenInRequest
+}
+
+// Tries Extractors in order until one returns a token string or an error occurs
+type MultiExtractor []Extractor
+
+func (e MultiExtractor) ExtractToken(req *http.Request) (string, error) {
+	// loop over header names and return the first one that contains data
+	for _, extractor := range e {
+		if tok, err := extractor.ExtractToken(req); tok != "" {
+			return tok, nil
+		} else if err != ErrNoTokenInRequest {
+			return "", err
+		}
+	}
+	return "", ErrNoTokenInRequest
+}
+
+// Wrap an Extractor in this to post-process the value before it's handed off.
+// See AuthorizationHeaderExtractor for an example
+type PostExtractionFilter struct {
+	Extractor
+	Filter func(string) (string, error)
+}
+
+func (e *PostExtractionFilter) ExtractToken(req *http.Request) (string, error) {
+	if tok, err := e.Extractor.ExtractToken(req); tok != "" {
+		return e.Filter(tok)
+	} else {
+		return "", err
+	}
+}

backend/vendor/github.com/dgrijalva/jwt-go/request/oauth2.go

@@ -0,0 +1,28 @@
+package request
+
+import (
+	"strings"
+)
+
+// Strips 'Bearer ' prefix from bearer token string
+func stripBearerPrefixFromTokenString(tok string) (string, error) {
+	// Should be a bearer token
+	if len(tok) > 6 && strings.ToUpper(tok[0:7]) == "BEARER " {
+		return tok[7:], nil
+	}
+	return tok, nil
+}
+
+// Extract bearer token from Authorization header
+// Uses PostExtractionFilter to strip "Bearer " prefix from header
+var AuthorizationHeaderExtractor = &PostExtractionFilter{
+	HeaderExtractor{"Authorization"},
+	stripBearerPrefixFromTokenString,
+}
+
+// Extractor for OAuth2 access tokens.  Looks in 'Authorization'
+// header then 'access_token' argument for a token.
+var OAuth2Extractor = &MultiExtractor{
+	AuthorizationHeaderExtractor,
+	ArgumentExtractor{"access_token"},
+}

backend/vendor/github.com/dgrijalva/jwt-go/request/request.go

@@ -0,0 +1,68 @@
+package request
+
+import (
+	"github.com/dgrijalva/jwt-go"
+	"net/http"
+)
+
+// Extract and parse a JWT token from an HTTP request.
+// This behaves the same as Parse, but accepts a request and an extractor
+// instead of a token string.  The Extractor interface allows you to define
+// the logic for extracting a token.  Several useful implementations are provided.
+//
+// You can provide options to modify parsing behavior
+func ParseFromRequest(req *http.Request, extractor Extractor, keyFunc jwt.Keyfunc, options ...ParseFromRequestOption) (token *jwt.Token, err error) {
+	// Create basic parser struct
+	p := &fromRequestParser{req, extractor, nil, nil}
+
+	// Handle options
+	for _, option := range options {
+		option(p)
+	}
+
+	// Set defaults
+	if p.claims == nil {
+		p.claims = jwt.MapClaims{}
+	}
+	if p.parser == nil {
+		p.parser = &jwt.Parser{}
+	}
+
+	// perform extract
+	tokenString, err := p.extractor.ExtractToken(req)
+	if err != nil {
+		return nil, err
+	}
+
+	// perform parse
+	return p.parser.ParseWithClaims(tokenString, p.claims, keyFunc)
+}
+
+// ParseFromRequest but with custom Claims type
+// DEPRECATED: use ParseFromRequest and the WithClaims option
+func ParseFromRequestWithClaims(req *http.Request, extractor Extractor, claims jwt.Claims, keyFunc jwt.Keyfunc) (token *jwt.Token, err error) {
+	return ParseFromRequest(req, extractor, keyFunc, WithClaims(claims))
+}
+
+type fromRequestParser struct {
+	req       *http.Request
+	extractor Extractor
+	claims    jwt.Claims
+	parser    *jwt.Parser
+}
+
+type ParseFromRequestOption func(*fromRequestParser)
+
+// Parse with custom claims
+func WithClaims(claims jwt.Claims) ParseFromRequestOption {
+	return func(p *fromRequestParser) {
+		p.claims = claims
+	}
+}
+
+// Parse using a custom parser
+func WithParser(parser *jwt.Parser) ParseFromRequestOption {
+	return func(p *fromRequestParser) {
+		p.parser = parser
+	}
+}

docker_init.sh

@@ -1,8 +1,14 @@
 #!/bin/sh
 
 # replace default api path to new one
-jspath=`ls /app/dist/js/app.*.js`
-cat ${jspath} | sed "s/localhost:8000/${CRAWLAB_API_ADDRESS}/g" > ${jspath}
+if [ "${CRAWLAB_API_ADDRESS}" = "" ]; 
+then
+	:
+else
+	jspath=`ls /app/dist/js/app.*.js`
+	cp ${jspath} ${jspath}.bak
+	sed -i "s/localhost:8000/${CRAWLAB_API_ADDRESS}/g" ${jspath}
+fi
 
 # start nginx
 service nginx start

examples/run_docker_master.sh

@@ -0,0 +1,10 @@
+docker run -d --rm --name crawlab \
+        -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \
+        -e CRAWLAB_MONGO_HOST=192.168.99.1 \
+        -e CRAWLAB_SERVER_MASTER=Y \
+        -e CRAWLAB_API_ADDRESS=192.168.99.100:8000 \
+        -e CRAWLAB_SPIDER_PATH=/app/spiders \
+        -p 8080:8080 \
+        -p 8000:8000 \
+        -v /var/logs/crawlab:/var/logs/crawlab \
+        tikazyq/crawlab:0.3.0

examples/run_docker_worker.sh

@@ -0,0 +1,6 @@
+docker run --rm --name crawlab \
+        -e CRAWLAB_REDIS_ADDRESS=192.168.99.1:6379 \
+        -e CRAWLAB_MONGO_HOST=192.168.99.1 \
+        -e CRAWLAB_SERVER_MASTER=N \
+        -v /var/logs/crawlab:/var/logs/crawlab \
+        tikazyq/crawlab:0.3.0

frontend/src/views/spider/SpiderList.vue

@@ -314,7 +314,8 @@ export default {
       console.log(value)
     },
     onAdd () {
-      this.addDialogVisible = true
+      // this.addDialogVisible = true
+      this.onAddCustomized()
     },
     onAddConfigurable () {
       this.$store.commit('spider/SET_SPIDER_FORM', {})
@@ -559,6 +560,13 @@ export default {
       background: rgba(103, 194, 58, .1);
       border: 1px solid rgba(103, 194, 58, .1);
     }
+
+    .add-spider-item.info {
+      color: #909399;
+      background: #f4f4f5;
+      border: 1px solid #e9e9eb;
+    }
+
   }
 
   .el-autocomplete {