From 25b4c7d272fa06e8013a8d87ff2a7c675994ce6b Mon Sep 17 00:00:00 2001 From: marvzhang Date: Mon, 17 Feb 2020 11:52:26 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90scrapy=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/services/scrapy.go | 11 +- .../github.com/Unknwon/goconfig/.gitignore | 3 + .../github.com/Unknwon/goconfig/LICENSE | 191 ++++++ .../github.com/Unknwon/goconfig/README.md | 68 +++ .../github.com/Unknwon/goconfig/README_ZH.md | 64 ++ .../github.com/Unknwon/goconfig/conf.go | 555 ++++++++++++++++++ .../github.com/Unknwon/goconfig/read.go | 294 ++++++++++ .../github.com/Unknwon/goconfig/write.go | 117 ++++ .../src/components/Scrapy/SpiderScrapy.vue | 29 +- frontend/src/views/spider/SpiderDetail.vue | 12 +- 10 files changed, 1333 insertions(+), 11 deletions(-) create mode 100644 backend/vendor/github.com/Unknwon/goconfig/.gitignore create mode 100644 backend/vendor/github.com/Unknwon/goconfig/LICENSE create mode 100644 backend/vendor/github.com/Unknwon/goconfig/README.md create mode 100644 backend/vendor/github.com/Unknwon/goconfig/README_ZH.md create mode 100644 backend/vendor/github.com/Unknwon/goconfig/conf.go create mode 100644 backend/vendor/github.com/Unknwon/goconfig/read.go create mode 100644 backend/vendor/github.com/Unknwon/goconfig/write.go diff --git a/backend/services/scrapy.go b/backend/services/scrapy.go index ebe615cd..c1e5158f 100644 --- a/backend/services/scrapy.go +++ b/backend/services/scrapy.go @@ -14,6 +14,7 @@ import ( "os/exec" "path" "runtime/debug" + "strconv" "strings" ) @@ -58,7 +59,6 @@ func GetScrapySettings(s model.Spider) (res []map[string]interface{}, err error) return res, err } - log.Infof(stdout.String()) if err := json.Unmarshal([]byte(stdout.String()), &res); err != nil { log.Errorf(err.Error()) debug.PrintStack() @@ -93,7 +93,9 @@ func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam case constants.String: line = fmt.Sprintf("%s = '%s'", param.Key, param.Value) case constants.Number: - line = fmt.Sprintf("%s = %s", param.Key, param.Value) + n := int64(param.Value.(float64)) + s := strconv.FormatInt(n, 10) + line = fmt.Sprintf("%s = %s", param.Key, s) case constants.Boolean: if param.Value.(bool) { line = fmt.Sprintf("%s = %s", param.Key, "True") @@ -111,8 +113,9 @@ func SaveScrapySettings(s model.Spider, settingsData []entity.ScrapySettingParam value := param.Value.(map[string]interface{}) var arr []string for k, v := range value { - str := v.(float64) - arr = append(arr, fmt.Sprintf("'%s': %.0f", k, str)) + n := int64(v.(float64)) + s := strconv.FormatInt(n, 10) + arr = append(arr, fmt.Sprintf("'%s': %s", k, s)) } line = fmt.Sprintf("%s = {%s}", param.Key, strings.Join(arr, ",")) } diff --git a/backend/vendor/github.com/Unknwon/goconfig/.gitignore b/backend/vendor/github.com/Unknwon/goconfig/.gitignore new file mode 100644 index 00000000..c81d5b37 --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/.gitignore @@ -0,0 +1,3 @@ +.DS_Store +*.iml +.idea \ No newline at end of file diff --git a/backend/vendor/github.com/Unknwon/goconfig/LICENSE b/backend/vendor/github.com/Unknwon/goconfig/LICENSE new file mode 100644 index 00000000..8405e89a --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/LICENSE @@ -0,0 +1,191 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/backend/vendor/github.com/Unknwon/goconfig/README.md b/backend/vendor/github.com/Unknwon/goconfig/README.md new file mode 100644 index 00000000..3a2218cb --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/README.md @@ -0,0 +1,68 @@ +goconfig [![Go Walker](http://gowalker.org/api/v1/badge)](http://gowalker.org/github.com/Unknwon/goconfig) +======== + +[中文文档](README_ZH.md) + +**IMPORTANT** + +- This library is under bug fix only mode, which means no more features will be added. +- I'm continuing working on better Go code with a different library: [ini](https://github.com/go-ini/ini). + +## About + +Package goconfig is a easy-use, comments-support configuration file parser for the Go Programming Language, which provides a structure similar to what you would find on Microsoft Windows INI files. + +The configuration file consists of sections, led by a `[section]` header and followed by `name:value` or `name=value` entries. Note that leading whitespace is removed from values. The optional values can contain format strings which refer to other values in the same section, or values in a special DEFAULT section. Comments are indicated by ";" or "#"; comments may begin anywhere on a single line. + +## Features + +- It simplified operation processes, easy to use and undersatnd; therefore, there are less chances to have errors. +- It uses exactly the same way to access a configuration file as you use Windows APIs, so you don't need to change your code style. +- It supports read recursion sections. +- It supports auto increment of key. +- It supports **READ** and **WRITE** configuration file with comments each section or key which all the other parsers don't support!!!!!!! +- It supports get value through type bool, float64, int, int64 and string, methods that start with "Must" means ignore errors and get zero-value if error occurs, or you can specify a default value. +- It's able to load multiple files to overwrite key values. + +## Installation + + go get github.com/unknwon/goconfig + +## API Documentation + +[Go Walker](http://gowalker.org/github.com/unknwon/goconfig). + +## Example + +Please see [conf.ini](testdata/conf.ini) as an example. + +### Usage + +- Function `LoadConfigFile` load file(s) depends on your situation, and return a variable with type `ConfigFile`. +- `GetValue` gives basic functionality of getting a value of given section and key. +- Methods like `Bool`, `Int`, `Int64` return corresponding type of values. +- Methods start with `Must` return corresponding type of values and returns zero-value of given type if something goes wrong. +- `SetValue` sets value to given section and key, and inserts somewhere if it does not exist. +- `DeleteKey` deletes by given section and key. +- Finally, `SaveConfigFile` saves your configuration to local file system. +- Use method `Reload` in case someone else modified your file(s). +- Methods contains `Comment` help you manipulate comments. +- `LoadFromReader` allows loading data without an intermediate file. +- `SaveConfigData` added, which writes configuration to an arbitrary writer. +- `ReloadData` allows to reload data from memory. + +Note that you cannot mix in-memory configuration with on-disk configuration. + +## More Information + +- All characters are CASE SENSITIVE, BE CAREFUL! + +## Credits + +- [goconf](http://code.google.com/p/goconf/) +- [robfig/config](https://github.com/robfig/config) +- [Delete an item from a slice](https://groups.google.com/forum/?fromgroups=#!topic/golang-nuts/lYz8ftASMQ0) + +## License + +This project is under Apache v2 License. See the [LICENSE](LICENSE) file for the full license text. diff --git a/backend/vendor/github.com/Unknwon/goconfig/README_ZH.md b/backend/vendor/github.com/Unknwon/goconfig/README_ZH.md new file mode 100644 index 00000000..d5fcbb95 --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/README_ZH.md @@ -0,0 +1,64 @@ +goconfig [![Build Status](https://drone.io/github.com/Unknwon/goconfig/status.png)](https://drone.io/github.com/Unknwon/goconfig/latest) [![Go Walker](http://gowalker.org/api/v1/badge)](http://gowalker.org/github.com/Unknwon/goconfig) +======== + +本库已被 [《Go名库讲解》](https://github.com/Unknwon/go-rock-libraries-showcases/tree/master/lectures/01-goconfig) 收录讲解，欢迎前往学习如何使用！ + +编码规范：基于 [Go 编码规范](https://github.com/Unknwon/go-code-convention) + +## 关于 + +包 goconfig 是一个易于使用，支持注释的 Go 语言配置文件解析器，该文件的书写格式和 Windows 下的 INI 文件一样。 + +配置文件由形为 `[section]` 的节构成，内部使用 `name:value` 或 `name=value` 这样的键值对；每行开头和尾部的空白符号都将被忽略；如果未指定任何节，则会默认放入名为 `DEFAULT` 的节当中；可以使用 “;” 或 “#” 来作为注释的开头，并可以放置于任意的单独一行中。 + +## 特性 + +- 简化流程，易于理解，更少出错。 +- 提供与 Windows API 一模一样的操作方式。 +- 支持读取递归节。 +- 支持自增键名。 +- 支持对注释的 **读** 和 **写** 操作，其它所有解析器都不支持！！！！ +- 可以直接返回 bool, float64, int, int64 和 string 类型的值，如果使用 “Must” 开头的方法，则一定会返回这个类型的一个值而不返回错误，如果错误发生则会返回零值。 +- 支持加载多个文件来重写值。 + +## 安装 + + go get github.com/Unknwon/goconfig + +或 + + gopm get github.com/Unknwon/goconfig + + +## API 文档 + +[Go Walker](http://gowalker.org/github.com/Unknwon/goconfig). + +## 示例 + +请查看 [conf.ini](testdata/conf.ini) 文件作为使用示例。 + +### 用例 + +- 函数 `LoadConfigFile` 加载一个或多个文件，然后返回一个类型为 `ConfigFile` 的变量。 +- `GetValue` 可以简单的获取某个值。 +- 像 `Bool`、`Int`、`Int64` 这样的方法会直接返回指定类型的值。 +- 以 `Must` 开头的方法不会返回错误，但当错误发生时会返回零值。 +- `SetValue` 可以设置某个值。 +- `DeleteKey` 可以删除某个键。 +- 最后，`SaveConfigFile` 可以保持您的配置到本地文件系统。 +- 使用方法 `Reload` 可以重载您的配置文件。 + +## 更多信息 + +- 所有字符都是大小写敏感的！ + +## 参考信息 + +- [goconf](http://code.google.com/p/goconf/) +- [robfig/config](https://github.com/robfig/config) +- [Delete an item from a slice](https://groups.google.com/forum/?fromgroups=#!topic/golang-nuts/lYz8ftASMQ0) + +## 授权许可 + +本项目采用 Apache v2 开源授权许可证，完整的授权说明已放置在 [LICENSE](LICENSE) 文件中。 diff --git a/backend/vendor/github.com/Unknwon/goconfig/conf.go b/backend/vendor/github.com/Unknwon/goconfig/conf.go new file mode 100644 index 00000000..dc19803e --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/conf.go @@ -0,0 +1,555 @@ +// Copyright 2013 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +// Package goconfig is a fully functional and comments-support configuration file(.ini) parser. +package goconfig + +import ( + "fmt" + "regexp" + "runtime" + "strconv" + "strings" + "sync" +) + +const ( + // Default section name. + DEFAULT_SECTION = "DEFAULT" + // Maximum allowed depth when recursively substituing variable names. + _DEPTH_VALUES = 200 +) + +type ParseError int + +const ( + ERR_SECTION_NOT_FOUND ParseError = iota + 1 + ERR_KEY_NOT_FOUND + ERR_BLANK_SECTION_NAME + ERR_COULD_NOT_PARSE +) + +var LineBreak = "\n" + +// Variable regexp pattern: %(variable)s +var varPattern = regexp.MustCompile(`%\(([^\)]+)\)s`) + +func init() { + if runtime.GOOS == "windows" { + LineBreak = "\r\n" + } +} + +// A ConfigFile represents a INI formar configuration file. +type ConfigFile struct { + lock sync.RWMutex // Go map is not safe. + fileNames []string // Support mutil-files. + data map[string]map[string]string // Section -> key : value + + // Lists can keep sections and keys in order. + sectionList []string // Section name list. + keyList map[string][]string // Section -> Key name list + + sectionComments map[string]string // Sections comments. + keyComments map[string]map[string]string // Keys comments. + BlockMode bool // Indicates whether use lock or not. +} + +// newConfigFile creates an empty configuration representation. +func newConfigFile(fileNames []string) *ConfigFile { + c := new(ConfigFile) + c.fileNames = fileNames + c.data = make(map[string]map[string]string) + c.keyList = make(map[string][]string) + c.sectionComments = make(map[string]string) + c.keyComments = make(map[string]map[string]string) + c.BlockMode = true + return c +} + +// SetValue adds a new section-key-value to the configuration. +// It returns true if the key and value were inserted, +// or returns false if the value was overwritten. +// If the section does not exist in advance, it will be created. +func (c *ConfigFile) SetValue(section, key, value string) bool { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + if len(key) == 0 { + return false + } + + if c.BlockMode { + c.lock.Lock() + defer c.lock.Unlock() + } + + // Check if section exists. + if _, ok := c.data[section]; !ok { + // Execute add operation. + c.data[section] = make(map[string]string) + // Append section to list. + c.sectionList = append(c.sectionList, section) + } + + // Check if key exists. + _, ok := c.data[section][key] + c.data[section][key] = value + if !ok { + // If not exists, append to key list. + c.keyList[section] = append(c.keyList[section], key) + } + return !ok +} + +// DeleteKey deletes the key in given section. +// It returns true if the key was deleted, +// or returns false if the section or key didn't exist. +func (c *ConfigFile) DeleteKey(section, key string) bool { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + if c.BlockMode { + c.lock.Lock() + defer c.lock.Unlock() + } + + // Check if section exists. + if _, ok := c.data[section]; !ok { + return false + } + + // Check if key exists. + if _, ok := c.data[section][key]; ok { + delete(c.data[section], key) + // Remove comments of key. + c.SetKeyComments(section, key, "") + // Get index of key. + i := 0 + for _, keyName := range c.keyList[section] { + if keyName == key { + break + } + i++ + } + // Remove from key list. + c.keyList[section] = append(c.keyList[section][:i], c.keyList[section][i+1:]...) + return true + } + return false +} + +// GetValue returns the value of key available in the given section. +// If the value needs to be unfolded +// (see e.g. %(google)s example in the GoConfig_test.go), +// then String does this unfolding automatically, up to +// _DEPTH_VALUES number of iterations. +// It returns an error and empty string value if the section does not exist, +// or key does not exist in DEFAULT and current sections. +func (c *ConfigFile) GetValue(section, key string) (string, error) { + if c.BlockMode { + c.lock.RLock() + defer c.lock.RUnlock() + } + + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + // Check if section exists + if _, ok := c.data[section]; !ok { + // Section does not exist. + return "", getError{ERR_SECTION_NOT_FOUND, section} + } + + // Section exists. + // Check if key exists or empty value. + value, ok := c.data[section][key] + if !ok { + // Check if it is a sub-section. + if i := strings.LastIndex(section, "."); i > -1 { + return c.GetValue(section[:i], key) + } + + // Return empty value. + return "", getError{ERR_KEY_NOT_FOUND, key} + } + + // Key exists. + var i int + for i = 0; i < _DEPTH_VALUES; i++ { + vr := varPattern.FindString(value) + if len(vr) == 0 { + break + } + + // Take off leading '%(' and trailing ')s'. + noption := strings.TrimLeft(vr, "%(") + noption = strings.TrimRight(noption, ")s") + + // Search variable in default section. + nvalue, err := c.GetValue(DEFAULT_SECTION, noption) + if err != nil && section != DEFAULT_SECTION { + // Search in the same section. + if _, ok := c.data[section][noption]; ok { + nvalue = c.data[section][noption] + } + } + + // Substitute by new value and take off leading '%(' and trailing ')s'. + value = strings.Replace(value, vr, nvalue, -1) + } + return value, nil +} + +// Bool returns bool type value. +func (c *ConfigFile) Bool(section, key string) (bool, error) { + value, err := c.GetValue(section, key) + if err != nil { + return false, err + } + return strconv.ParseBool(value) +} + +// Float64 returns float64 type value. +func (c *ConfigFile) Float64(section, key string) (float64, error) { + value, err := c.GetValue(section, key) + if err != nil { + return 0.0, err + } + return strconv.ParseFloat(value, 64) +} + +// Int returns int type value. +func (c *ConfigFile) Int(section, key string) (int, error) { + value, err := c.GetValue(section, key) + if err != nil { + return 0, err + } + return strconv.Atoi(value) +} + +// Int64 returns int64 type value. +func (c *ConfigFile) Int64(section, key string) (int64, error) { + value, err := c.GetValue(section, key) + if err != nil { + return 0, err + } + return strconv.ParseInt(value, 10, 64) +} + +// MustValue always returns value without error. +// It returns empty string if error occurs, or the default value if given. +func (c *ConfigFile) MustValue(section, key string, defaultVal ...string) string { + val, err := c.GetValue(section, key) + if len(defaultVal) > 0 && (err != nil || len(val) == 0) { + return defaultVal[0] + } + return val +} + +// MustValueSet always returns value without error, +// It returns empty string if error occurs, or the default value if given, +// and a bool value indicates whether default value is returned. +func (c *ConfigFile) MustValueSet(section, key string, defaultVal ...string) (string, bool) { + val, err := c.GetValue(section, key) + if len(defaultVal) > 0 && (err != nil || len(val) == 0) { + c.SetValue(section, key, defaultVal[0]) + return defaultVal[0], true + } + return val, false +} + +// MustValueRange always returns value without error, +// it returns default value if error occurs or doesn't fit into range. +func (c *ConfigFile) MustValueRange(section, key, defaultVal string, candidates []string) string { + val, err := c.GetValue(section, key) + if err != nil || len(val) == 0 { + return defaultVal + } + + for _, cand := range candidates { + if val == cand { + return val + } + } + return defaultVal +} + +// MustValueArray always returns value array without error, +// it returns empty array if error occurs, split by delimiter otherwise. +func (c *ConfigFile) MustValueArray(section, key, delim string) []string { + val, err := c.GetValue(section, key) + if err != nil || len(val) == 0 { + return []string{} + } + + vals := strings.Split(val, delim) + for i := range vals { + vals[i] = strings.TrimSpace(vals[i]) + } + return vals +} + +// MustBool always returns value without error, +// it returns false if error occurs. +func (c *ConfigFile) MustBool(section, key string, defaultVal ...bool) bool { + val, err := c.Bool(section, key) + if len(defaultVal) > 0 && err != nil { + return defaultVal[0] + } + return val +} + +// MustFloat64 always returns value without error, +// it returns 0.0 if error occurs. +func (c *ConfigFile) MustFloat64(section, key string, defaultVal ...float64) float64 { + value, err := c.Float64(section, key) + if len(defaultVal) > 0 && err != nil { + return defaultVal[0] + } + return value +} + +// MustInt always returns value without error, +// it returns 0 if error occurs. +func (c *ConfigFile) MustInt(section, key string, defaultVal ...int) int { + value, err := c.Int(section, key) + if len(defaultVal) > 0 && err != nil { + return defaultVal[0] + } + return value +} + +// MustInt64 always returns value without error, +// it returns 0 if error occurs. +func (c *ConfigFile) MustInt64(section, key string, defaultVal ...int64) int64 { + value, err := c.Int64(section, key) + if len(defaultVal) > 0 && err != nil { + return defaultVal[0] + } + return value +} + +// GetSectionList returns the list of all sections +// in the same order in the file. +func (c *ConfigFile) GetSectionList() []string { + list := make([]string, len(c.sectionList)) + copy(list, c.sectionList) + return list +} + +// GetKeyList returns the list of all keys in give section +// in the same order in the file. +// It returns nil if given section does not exist. +func (c *ConfigFile) GetKeyList(section string) []string { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + if c.BlockMode { + c.lock.RLock() + defer c.lock.RUnlock() + } + + // Check if section exists. + if _, ok := c.data[section]; !ok { + return nil + } + + // Non-default section has a blank key as section keeper. + list := make([]string, 0, len(c.keyList[section])) + for _, key := range c.keyList[section] { + if key != " " { + list = append(list, key) + } + } + return list +} + +// DeleteSection deletes the entire section by given name. +// It returns true if the section was deleted, and false if the section didn't exist. +func (c *ConfigFile) DeleteSection(section string) bool { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + if c.BlockMode { + c.lock.Lock() + defer c.lock.Unlock() + } + + // Check if section exists. + if _, ok := c.data[section]; !ok { + return false + } + + delete(c.data, section) + // Remove comments of section. + c.SetSectionComments(section, "") + // Get index of section. + i := 0 + for _, secName := range c.sectionList { + if secName == section { + break + } + i++ + } + // Remove from section and key list. + c.sectionList = append(c.sectionList[:i], c.sectionList[i+1:]...) + delete(c.keyList, section) + return true +} + +// GetSection returns key-value pairs in given section. +// If section does not exist, returns nil and error. +func (c *ConfigFile) GetSection(section string) (map[string]string, error) { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + if c.BlockMode { + c.lock.Lock() + defer c.lock.Unlock() + } + + // Check if section exists. + if _, ok := c.data[section]; !ok { + // Section does not exist. + return nil, getError{ERR_SECTION_NOT_FOUND, section} + } + + // Remove pre-defined key. + secMap := c.data[section] + delete(c.data[section], " ") + + // Section exists. + return secMap, nil +} + +// SetSectionComments adds new section comments to the configuration. +// If comments are empty(0 length), it will remove its section comments! +// It returns true if the comments were inserted or removed, +// or returns false if the comments were overwritten. +func (c *ConfigFile) SetSectionComments(section, comments string) bool { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + if len(comments) == 0 { + if _, ok := c.sectionComments[section]; ok { + delete(c.sectionComments, section) + } + + // Not exists can be seen as remove. + return true + } + + // Check if comments exists. + _, ok := c.sectionComments[section] + if comments[0] != '#' && comments[0] != ';' { + comments = "; " + comments + } + c.sectionComments[section] = comments + return !ok +} + +// SetKeyComments adds new section-key comments to the configuration. +// If comments are empty(0 length), it will remove its section-key comments! +// It returns true if the comments were inserted or removed, +// or returns false if the comments were overwritten. +// If the section does not exist in advance, it is created. +func (c *ConfigFile) SetKeyComments(section, key, comments string) bool { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + // Check if section exists. + if _, ok := c.keyComments[section]; ok { + if len(comments) == 0 { + if _, ok := c.keyComments[section][key]; ok { + delete(c.keyComments[section], key) + } + + // Not exists can be seen as remove. + return true + } + } else { + if len(comments) == 0 { + // Not exists can be seen as remove. + return true + } else { + // Execute add operation. + c.keyComments[section] = make(map[string]string) + } + } + + // Check if key exists. + _, ok := c.keyComments[section][key] + if comments[0] != '#' && comments[0] != ';' { + comments = "; " + comments + } + c.keyComments[section][key] = comments + return !ok +} + +// GetSectionComments returns the comments in the given section. +// It returns an empty string(0 length) if the comments do not exist. +func (c *ConfigFile) GetSectionComments(section string) (comments string) { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + return c.sectionComments[section] +} + +// GetKeyComments returns the comments of key in the given section. +// It returns an empty string(0 length) if the comments do not exist. +func (c *ConfigFile) GetKeyComments(section, key string) (comments string) { + // Blank section name represents DEFAULT section. + if len(section) == 0 { + section = DEFAULT_SECTION + } + + if _, ok := c.keyComments[section]; ok { + return c.keyComments[section][key] + } + return "" +} + +// getError occurs when get value in configuration file with invalid parameter. +type getError struct { + Reason ParseError + Name string +} + +// Error implements Error interface. +func (err getError) Error() string { + switch err.Reason { + case ERR_SECTION_NOT_FOUND: + return fmt.Sprintf("section '%s' not found", err.Name) + case ERR_KEY_NOT_FOUND: + return fmt.Sprintf("key '%s' not found", err.Name) + } + return "invalid get error" +} diff --git a/backend/vendor/github.com/Unknwon/goconfig/read.go b/backend/vendor/github.com/Unknwon/goconfig/read.go new file mode 100644 index 00000000..37990fb6 --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/read.go @@ -0,0 +1,294 @@ +// Copyright 2013 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package goconfig + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "strings" + "time" +) + +// Read reads an io.Reader and returns a configuration representation. +// This representation can be queried with GetValue. +func (c *ConfigFile) read(reader io.Reader) (err error) { + buf := bufio.NewReader(reader) + + // Handle BOM-UTF8. + // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding + mask, err := buf.Peek(3) + if err == nil && len(mask) >= 3 && + mask[0] == 239 && mask[1] == 187 && mask[2] == 191 { + buf.Read(mask) + } + + count := 1 // Counter for auto increment. + // Current section name. + section := DEFAULT_SECTION + var comments string + // Parse line-by-line + for { + line, err := buf.ReadString('\n') + line = strings.TrimSpace(line) + lineLengh := len(line) //[SWH|+] + if err != nil { + if err != io.EOF { + return err + } + + // Reached end of file, if nothing to read then break, + // otherwise handle the last line. + if lineLengh == 0 { + break + } + } + + // switch written for readability (not performance) + switch { + case lineLengh == 0: // Empty line + continue + case line[0] == '#' || line[0] == ';': // Comment + // Append comments + if len(comments) == 0 { + comments = line + } else { + comments += LineBreak + line + } + continue + case line[0] == '[' && line[lineLengh-1] == ']': // New section. + // Get section name. + section = strings.TrimSpace(line[1 : lineLengh-1]) + // Set section comments and empty if it has comments. + if len(comments) > 0 { + c.SetSectionComments(section, comments) + comments = "" + } + // Make section exist even though it does not have any key. + c.SetValue(section, " ", " ") + // Reset counter. + count = 1 + continue + case section == "": // No section defined so far + return readError{ERR_BLANK_SECTION_NAME, line} + default: // Other alternatives + var ( + i int + keyQuote string + key string + valQuote string + value string + ) + //[SWH|+]:支持引号包围起来的字串 + if line[0] == '"' { + if lineLengh >= 6 && line[0:3] == `"""` { + keyQuote = `"""` + } else { + keyQuote = `"` + } + } else if line[0] == '`' { + keyQuote = "`" + } + if keyQuote != "" { + qLen := len(keyQuote) + pos := strings.Index(line[qLen:], keyQuote) + if pos == -1 { + return readError{ERR_COULD_NOT_PARSE, line} + } + pos = pos + qLen + i = strings.IndexAny(line[pos:], "=:") + if i <= 0 { + return readError{ERR_COULD_NOT_PARSE, line} + } + i = i + pos + key = line[qLen:pos] //保留引号内的两端的空格 + } else { + i = strings.IndexAny(line, "=:") + if i <= 0 { + return readError{ERR_COULD_NOT_PARSE, line} + } + key = strings.TrimSpace(line[0:i]) + } + //[SWH|+]; + + // Check if it needs auto increment. + if key == "-" { + key = "#" + fmt.Sprint(count) + count++ + } + + //[SWH|+]:支持引号包围起来的字串 + lineRight := strings.TrimSpace(line[i+1:]) + lineRightLength := len(lineRight) + firstChar := "" + if lineRightLength >= 2 { + firstChar = lineRight[0:1] + } + if firstChar == "`" { + valQuote = "`" + } else if lineRightLength >= 6 && lineRight[0:3] == `"""` { + valQuote = `"""` + } + if valQuote != "" { + qLen := len(valQuote) + pos := strings.LastIndex(lineRight[qLen:], valQuote) + if pos == -1 { + return readError{ERR_COULD_NOT_PARSE, line} + } + pos = pos + qLen + value = lineRight[qLen:pos] + } else { + value = strings.TrimSpace(lineRight[0:]) + } + //[SWH|+]; + + c.SetValue(section, key, value) + // Set key comments and empty if it has comments. + if len(comments) > 0 { + c.SetKeyComments(section, key, comments) + comments = "" + } + } + + // Reached end of file. + if err == io.EOF { + break + } + } + return nil +} + +// LoadFromData accepts raw data directly from memory +// and returns a new configuration representation. +// Note that the configuration is written to the system +// temporary folder, so your file should not contain +// sensitive information. +func LoadFromData(data []byte) (c *ConfigFile, err error) { + // Save memory data to temporary file to support further operations. + tmpName := path.Join(os.TempDir(), "goconfig", fmt.Sprintf("%d", time.Now().Nanosecond())) + if err = os.MkdirAll(path.Dir(tmpName), os.ModePerm); err != nil { + return nil, err + } + if err = ioutil.WriteFile(tmpName, data, 0655); err != nil { + return nil, err + } + + c = newConfigFile([]string{tmpName}) + err = c.read(bytes.NewBuffer(data)) + return c, err +} + +// LoadFromReader accepts raw data directly from a reader +// and returns a new configuration representation. +// You must use ReloadData to reload. +// You cannot append files a configfile read this way. +func LoadFromReader(in io.Reader) (c *ConfigFile, err error) { + c = newConfigFile([]string{""}) + err = c.read(in) + return c, err +} + +func (c *ConfigFile) loadFile(fileName string) (err error) { + f, err := os.Open(fileName) + if err != nil { + return err + } + defer f.Close() + + return c.read(f) +} + +// LoadConfigFile reads a file and returns a new configuration representation. +// This representation can be queried with GetValue. +func LoadConfigFile(fileName string, moreFiles ...string) (c *ConfigFile, err error) { + // Append files' name together. + fileNames := make([]string, 1, len(moreFiles)+1) + fileNames[0] = fileName + if len(moreFiles) > 0 { + fileNames = append(fileNames, moreFiles...) + } + + c = newConfigFile(fileNames) + + for _, name := range fileNames { + if err = c.loadFile(name); err != nil { + return nil, err + } + } + + return c, nil +} + +// Reload reloads configuration file in case it has changes. +func (c *ConfigFile) Reload() (err error) { + var cfg *ConfigFile + if len(c.fileNames) == 1 { + if c.fileNames[0] == "" { + return fmt.Errorf("file opened from in-memory data, use ReloadData to reload") + } + cfg, err = LoadConfigFile(c.fileNames[0]) + } else { + cfg, err = LoadConfigFile(c.fileNames[0], c.fileNames[1:]...) + } + + if err == nil { + *c = *cfg + } + return err +} + +// ReloadData reloads configuration file from memory +func (c *ConfigFile) ReloadData(in io.Reader) (err error) { + var cfg *ConfigFile + if len(c.fileNames) != 1 { + return fmt.Errorf("Multiple files loaded, unable to mix in-memory and file data") + } + + cfg, err = LoadFromReader(in) + if err == nil { + *c = *cfg + } + return err +} + +// AppendFiles appends more files to ConfigFile and reload automatically. +func (c *ConfigFile) AppendFiles(files ...string) error { + if len(c.fileNames) == 1 && c.fileNames[0] == "" { + return fmt.Errorf("Cannot append file data to in-memory data") + } + c.fileNames = append(c.fileNames, files...) + return c.Reload() +} + +// readError occurs when read configuration file with wrong format. +type readError struct { + Reason ParseError + Content string // Line content +} + +// Error implement Error interface. +func (err readError) Error() string { + switch err.Reason { + case ERR_BLANK_SECTION_NAME: + return "empty section name not allowed" + case ERR_COULD_NOT_PARSE: + return fmt.Sprintf("could not parse line: %s", string(err.Content)) + } + return "invalid read error" +} diff --git a/backend/vendor/github.com/Unknwon/goconfig/write.go b/backend/vendor/github.com/Unknwon/goconfig/write.go new file mode 100644 index 00000000..af36d1ad --- /dev/null +++ b/backend/vendor/github.com/Unknwon/goconfig/write.go @@ -0,0 +1,117 @@ +// Copyright 2013 Unknwon +// +// Licensed under the Apache License, Version 2.0 (the "License"): you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package goconfig + +import ( + "bytes" + "io" + "os" + "strings" +) + +// Write spaces around "=" to look better. +var PrettyFormat = true + +// SaveConfigData writes configuration to a writer +func SaveConfigData(c *ConfigFile, out io.Writer) (err error) { + equalSign := "=" + if PrettyFormat { + equalSign = " = " + } + + buf := bytes.NewBuffer(nil) + for _, section := range c.sectionList { + // Write section comments. + if len(c.GetSectionComments(section)) > 0 { + if _, err = buf.WriteString(c.GetSectionComments(section) + LineBreak); err != nil { + return err + } + } + + if section != DEFAULT_SECTION { + // Write section name. + if _, err = buf.WriteString("[" + section + "]" + LineBreak); err != nil { + return err + } + } + + for _, key := range c.keyList[section] { + if key != " " { + // Write key comments. + if len(c.GetKeyComments(section, key)) > 0 { + if _, err = buf.WriteString(c.GetKeyComments(section, key) + LineBreak); err != nil { + return err + } + } + + keyName := key + // Check if it's auto increment. + if keyName[0] == '#' { + keyName = "-" + } + //[SWH|+]:支持键名包含等号和冒号 + if strings.Contains(keyName, `=`) || strings.Contains(keyName, `:`) { + if strings.Contains(keyName, "`") { + if strings.Contains(keyName, `"`) { + keyName = `"""` + keyName + `"""` + } else { + keyName = `"` + keyName + `"` + } + } else { + keyName = "`" + keyName + "`" + } + } + value := c.data[section][key] + // In case key value contains "`" or "\"". + if strings.Contains(value, "`") { + if strings.Contains(value, `"`) { + value = `"""` + value + `"""` + } else { + value = `"` + value + `"` + } + } + + // Write key and value. + if _, err = buf.WriteString(keyName + equalSign + value + LineBreak); err != nil { + return err + } + } + } + + // Put a line between sections. + if _, err = buf.WriteString(LineBreak); err != nil { + return err + } + } + + if _, err := buf.WriteTo(out); err != nil { + return err + } + return nil +} + +// SaveConfigFile writes configuration file to local file system +func SaveConfigFile(c *ConfigFile, filename string) (err error) { + // Write configuration file by filename. + var f *os.File + if f, err = os.Create(filename); err != nil { + return err + } + + if err := SaveConfigData(c, f); err != nil { + return err + } + return f.Close() +} diff --git a/frontend/src/components/Scrapy/SpiderScrapy.vue b/frontend/src/components/Scrapy/SpiderScrapy.vue index 0843b8ca..8a752fc9 100644 --- a/frontend/src/components/Scrapy/SpiderScrapy.vue +++ b/frontend/src/components/Scrapy/SpiderScrapy.vue @@ -95,16 +95,18 @@ :data="spiderScrapySettings" border :header-cell-style="{background:'rgb(48, 65, 86)',color:'white'}" + max-height="calc(100vh - 240px" > @@ -113,7 +115,7 @@ width="120px" >