import Vue from 'vue' import request from '../../api/request' const state = { // list of spiders spiderList: [], spiderTotal: 0, // active spider data spiderForm: {}, // spider scrapy settings spiderScrapySettings: [], // spider scrapy items spiderScrapyItems: [], // spider scrapy pipelines spiderScrapyPipelines: [], // node to deploy/run activeNode: {}, // upload form for importing spiders importForm: { url: '', type: 'github' }, // spider overview stats overviewStats: {}, // spider status stats statusStats: [], // spider daily stats dailyStats: [], // spider node stats nodeStats: [], // filters filterSite: '', // preview crawl data previewCrawlData: [], // template list templateList: [], // spider file tree fileTree: {}, // config list ts configListTs: undefined } const getters = {} const mutations = { SET_SPIDER_TOTAL (state, value) { state.spiderTotal = value }, SET_SPIDER_FORM (state, value) { state.spiderForm = value }, SET_SPIDER_LIST (state, value) { state.spiderList = value }, SET_ACTIVE_NODE (state, value) { state.activeNode = value }, SET_IMPORT_FORM (state, value) { state.importForm = value }, SET_OVERVIEW_STATS (state, value) { state.overviewStats = value }, SET_STATUS_STATS (state, value) { state.statusStats = value }, SET_DAILY_STATS (state, value) { state.dailyStats = value }, SET_NODE_STATS (state, value) { state.nodeStats = value }, SET_FILTER_SITE (state, value) { state.filterSite = value }, SET_PREVIEW_CRAWL_DATA (state, value) { state.previewCrawlData = value }, SET_SPIDER_FORM_CONFIG_SETTINGS (state, payload) { const settings = {} payload.forEach(row => { settings[row.name] = row.value }) Vue.set(state.spiderForm.config, 'settings', settings) }, SET_TEMPLATE_LIST (state, value) { state.templateList = value }, SET_FILE_TREE (state, value) { state.fileTree = value }, SET_SPIDER_SCRAPY_SETTINGS (state, value) { state.spiderScrapySettings = value }, SET_SPIDER_SCRAPY_ITEMS (state, value) { state.spiderScrapyItems = value }, SET_SPIDER_SCRAPY_PIPELINES (state, value) { state.spiderScrapyPipelines = value }, SET_CONFIG_LIST_TS (state, value) { state.configListTs = value } } const actions = { getSpiderList ({ state, commit }, params = {}) { return request.get('/spiders', params) .then(response => { commit('SET_SPIDER_LIST', response.data.data.list) commit('SET_SPIDER_TOTAL', response.data.data.total) }) }, editSpider ({ state, dispatch }) { return request.post(`/spiders/${state.spiderForm._id}`, state.spiderForm) }, deleteSpider ({ state, dispatch }, id) { return request.delete(`/spiders/${id}`) }, getSpiderData ({ state, commit }, id) { return request.get(`/spiders/${id}`) .then(response => { let data = response.data.data commit('SET_SPIDER_FORM', data) }) }, async getSpiderScrapySpiders ({ state, commit }, id) { const res = await request.get(`/spiders/${id}/scrapy/spiders`) state.spiderForm.spider_names = res.data.data commit('SET_SPIDER_FORM', state.spiderForm) }, async getSpiderScrapySettings ({ state, commit }, id) { const res = await request.get(`/spiders/${id}/scrapy/settings`) commit('SET_SPIDER_SCRAPY_SETTINGS', res.data.data.map(d => { const key = d.key const value = d.value let type = typeof value if (type === 'object') { if (Array.isArray(value)) { type = 'array' } else { type = 'object' } } return { key, value, type } })) }, async saveSpiderScrapySettings ({ state }, id) { return request.post(`/spiders/${id}/scrapy/settings`, state.spiderScrapySettings) }, async getSpiderScrapyItems ({ state, commit }, id) { const res = await request.get(`/spiders/${id}/scrapy/items`) let nodeId = 0 commit('SET_SPIDER_SCRAPY_ITEMS', res.data.data.map(d => { d.id = nodeId++ d.label = d.name d.level = 1 d.isEdit = false d.children = d.fields.map(f => { return { id: nodeId++, label: f, level: 2, isEdit: false } }) return d })) }, async saveSpiderScrapyItems ({ state }, id) { return request.post(`/spiders/${id}/scrapy/items`, state.spiderScrapyItems.map(d => { d.name = d.label d.fields = d.children.map(f => f.label) return d })) }, async getSpiderScrapyPipelines ({ state, commit }, id) { const res = await request.get(`/spiders/${id}/scrapy/pipelines`) commit('SET_SPIDER_SCRAPY_PIPELINES', res.data.data) }, async saveSpiderScrapyPipelines ({ state }, id) { return request.post(`/spiders/${id}/scrapy/pipelines`, state.spiderScrapyPipelines) }, async getSpiderScrapySpiderFilepath ({ state, commit }, payload) { const { id, spiderName } = payload return request.get(`/spiders/${id}/scrapy/spider/filepath`, { spider_name: spiderName }) }, addSpiderScrapySpider ({ state }, payload) { const { id, form } = payload return request.put(`/spiders/${id}/scrapy/spiders`, form) }, crawlSpider ({ state, dispatch }, payload) { const { spiderId, runType, nodeIds, param } = payload return request.put(`/tasks`, { spider_id: spiderId, run_type: runType, node_ids: nodeIds, param: param }) }, crawlSelectedSpiders ({ state, dispatch }, payload) { const { taskParams, runType, nodeIds } = payload return request.post(`/spiders-run`, { task_params: taskParams, run_type: runType, node_ids: nodeIds }) }, getTaskList ({ state, commit }, id) { return request.get(`/spiders/${id}/tasks`) .then(response => { commit('task/SET_TASK_LIST', response.data.data ? response.data.data.map(d => { return d }).sort((a, b) => a.create_ts < b.create_ts ? 1 : -1) : [], { root: true }) }) }, getDir ({ state, commit }, path) { const id = state.spiderForm._id return request.get(`/spiders/${id}/dir`) .then(response => { commit('') }) }, importGithub ({ state }) { const url = state.importForm.url return request.post('/spiders/import/github', { url }) }, getSpiderStats ({ state, commit }) { return request.get(`/spiders/${state.spiderForm._id}/stats`) .then(response => { commit('SET_OVERVIEW_STATS', response.data.data.overview) // commit('SET_STATUS_STATS', response.data.task_count_by_status) commit('SET_DAILY_STATS', response.data.data.daily) // commit('SET_NODE_STATS', response.data.task_count_by_node) }) }, getPreviewCrawlData ({ state, commit }) { return request.post(`/spiders/${state.spiderForm._id}/preview_crawl`) .then(response => { commit('SET_PREVIEW_CRAWL_DATA', response.data.items) }) }, extractFields ({ state, commit }) { return request.post(`/spiders/${state.spiderForm._id}/extract_fields`) }, postConfigSpiderConfig ({ state }) { return request.post(`/config_spiders/${state.spiderForm._id}/config`, state.spiderForm.config) }, saveConfigSpiderSpiderfile ({ state, rootState }) { const content = rootState.file.fileContent return request.post(`/config_spiders/${state.spiderForm._id}/spiderfile`, { content }) }, addConfigSpider ({ state }) { return request.put(`/config_spiders`, state.spiderForm) }, addSpider ({ state }) { return request.put(`/spiders`, state.spiderForm) }, async getTemplateList ({ state, commit }) { const res = await request.get(`/config_spiders_templates`) commit('SET_TEMPLATE_LIST', res.data.data) }, async getScheduleList ({ state, commit }, payload) { const { id } = payload const res = await request.get(`/spiders/${id}/schedules`) commit('schedule/SET_SCHEDULE_LIST', res.data.data, { root: true }) }, async getFileTree ({ state, commit }, payload) { const id = payload ? payload.id : state.spiderForm._id const res = await request.get(`/spiders/${id}/file/tree`) commit('SET_FILE_TREE', res.data.data) } } export default { namespaced: true, state, getters, mutations, actions }