ad4567fc01
这两批改动此前一直压在工作区未提交，本次一并归档。go build ./... 通过。
一、招聘爬虫健壮性
- 新增 internal/service/recruitment/source_config.go：
crawl_source.config（VARCHAR(1000) JSON）的源级抓取参数解析 —— 增量窗口天数、
单次详情页上限、最大翻页数、详情间隔限速、自定义标准词/排除词。
全部字段可选，JSON 解析失败即回退默认值，保证历史数据零迁移可用，
且一个源的脏配置不会拖垮整个调度。
- 新增 internal/service/recruitment/keywords.go（含 keywords_test.go 与 testdata/）：
链接筛选由「URL 形态命中 AND 文本语义命中」改为评分制 —— URL 形态加分、
标准词加分、排除词大幅减分，达阈值即入选。原与逻辑会系统性漏抓
「补充工作人员的通知」「公开选调公务员简章」等标题变体，且静默无报错。
- 新增 manifest/sql/recruitment/004_crawl_source_status.sql：
crawl_source 增加「最近一次运行状态」冗余列，抓取结束时写入，
使 Sources() 查询零 JOIN 零扫描 —— 规避原实现全表扫描只增不减的 crawl_log、
随运行时间线性劣化的问题。幂等，可重复执行。
- internal/service/recruitment/crawler.go：在既有「静态两级 / SPA / curl 回退」
流程上做锚点抽取与过滤、编码回退的健壮性增强。
- internal/service/recruitment/bark.go、recruitment.go：配合上述调整。
二、通知记录增强
- api/notice、internal/controller/notice、internal/service/notice、
internal/model/{dto,entity,do}/notice.go：通知日志查询与操作扩展 ——
批次号、状态、重试次数、来源、耗时、详情、删除、清空。
- 新增 internal/model/dto/notice_meta.go（字典选项元数据，供前端筛选器取选项）。
- 新增 manifest/sql/017_notice_log_enhance.sql：notice_log 扩展列 +
删除/清空/详情/字典选项接口权限，幂等（ALTER 走 information_schema 判断，
菜单 ON DUPLICATE KEY UPDATE，角色绑定 INSERT IGNORE）。
三、仓库卫生
- .gitignore 补 rc_*.log 与 rc_verify*，挡掉本地离线验证产物
（含 30MB 的 rc_verify.exe），避免误入库。
设计依据见 docs/recruitment-crawler-design.md。
注：017 与 recruitment/004 两个 SQL 为增量迁移，DB 结构变更不自动 DDL，
生产库需手动导入。
77 lines
6.2 KiB
Go
77 lines
6.2 KiB
Go
-- 017_notice_log_enhance.sql
|
||
-- 通知历史记录增强：notice_log 扩展字段（批次/类型统计/耗时/来源/重试/备注）
|
||
-- + 删除/清空/详情 接口权限 + 字典选项接口权限。
|
||
-- 幂等：ALTER TABLE ADD COLUMN 用 information_schema 判断（MySQL 8 不支持 COLUMN IF NOT EXISTS）；
|
||
-- 菜单用 INSERT ... ON DUPLICATE KEY UPDATE；角色绑定用 INSERT IGNORE。
|
||
-- 依赖：016_user_notice_job.sql 已执行（notice_log / admin_menu / admin_role_menu 已存在）。
|
||
|
||
-- ============ 1) notice_log 扩展字段 ============
|
||
-- batch_id：同一次业务触发（如一次任务失败同时发 bark + pushplus）共用，便于按批次追溯
|
||
SET @has_col := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND COLUMN_NAME='batch_id');
|
||
SET @sql := IF(@has_col=0, 'ALTER TABLE notice_log ADD COLUMN batch_id VARCHAR(32) NOT NULL DEFAULT '''' COMMENT ''批次号(同一次业务触发)'' AFTER id', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- status：0待发送 1成功 2失败（result 为旧字段 1成功 0失败，保留兼容）
|
||
SET @has_col := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND COLUMN_NAME='status');
|
||
SET @sql := IF(@has_col=0, 'ALTER TABLE notice_log ADD COLUMN status TINYINT NOT NULL DEFAULT 1 COMMENT ''0待发送 1成功 2失败'' AFTER body', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- retry_count：已重试次数
|
||
SET @has_col := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND COLUMN_NAME='retry_count');
|
||
SET @sql := IF(@has_col=0, 'ALTER TABLE notice_log ADD COLUMN retry_count INT NOT NULL DEFAULT 0 COMMENT ''已重试次数'' AFTER error', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- duration_ms：渠道接口耗时
|
||
SET @has_col := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND COLUMN_NAME='duration_ms');
|
||
SET @sql := IF(@has_col=0, 'ALTER TABLE notice_log ADD COLUMN duration_ms INT NOT NULL DEFAULT 0 COMMENT ''渠道接口耗时(毫秒)'' AFTER retry_count', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- source：触发来源 auto/manual/test
|
||
SET @has_col := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND COLUMN_NAME='source');
|
||
SET @sql := IF(@has_col=0, 'ALTER TABLE notice_log ADD COLUMN source VARCHAR(20) NOT NULL DEFAULT ''auto'' COMMENT ''触发来源 auto/manual/test'' AFTER duration_ms', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- remark：备注
|
||
SET @has_col := (SELECT COUNT(*) FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND COLUMN_NAME='remark');
|
||
SET @sql := IF(@has_col=0, 'ALTER TABLE notice_log ADD COLUMN remark VARCHAR(500) NOT NULL DEFAULT '''' COMMENT ''备注'' AFTER source', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- 索引：渠道+时间、状态+时间、批次
|
||
SET @has_idx := (SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND INDEX_NAME='idx_channel');
|
||
SET @sql := IF(@has_idx=0, 'ALTER TABLE notice_log ADD INDEX idx_channel (channel_code, created_at)', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
SET @has_idx := (SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND INDEX_NAME='idx_status');
|
||
SET @sql := IF(@has_idx=0, 'ALTER TABLE notice_log ADD INDEX idx_status (status, created_at)', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
SET @has_idx := (SELECT COUNT(*) FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME='notice_log' AND INDEX_NAME='idx_batch');
|
||
SET @sql := IF(@has_idx=0, 'ALTER TABLE notice_log ADD INDEX idx_batch (batch_id)', 'SELECT 1');
|
||
PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||
|
||
-- 历史数据回填：result=1 → status=1；result=0 且有错误 → status=2
|
||
UPDATE notice_log SET status=1 WHERE status=0 AND result=1;
|
||
UPDATE notice_log SET status=2 WHERE status=0 AND result=0 AND error <> '';
|
||
UPDATE notice_log SET source='test' WHERE event_type='test' AND source='auto';
|
||
|
||
-- ============ 2) 菜单：通知历史记录（替换原「通知日志」并调整排序） ============
|
||
-- 982 原为「通知日志」，此处改名为「通知历史」以贴合需求（保留 id 以延续角色绑定）
|
||
UPDATE admin_menu SET name='通知历史', icon='mdi:history', component='system/notice/log/index', permission='notice:log', sort=3
|
||
WHERE id=982 AND type=1;
|
||
|
||
-- ============ 3) type=2 API 权限：通知历史记录 ============
|
||
-- 路径必须与 api/notice/notice.go 的 g.Meta path 完全一致（含 /api/service/admin 前缀）
|
||
INSERT INTO admin_menu (id, parent_id, name, icon, type, path, component, permission, sort, status, hidden, created_at, updated_at) VALUES
|
||
(9821, 982, '查询', '', 2, 'POST /api/service/admin/notice/log/list', '', 'notice:log:list', 1, 1, 0, NOW(), NOW()),
|
||
(9822, 982, '详情', '', 2, 'POST /api/service/admin/notice/log/detail', '', 'notice:log:detail', 2, 1, 0, NOW(), NOW()),
|
||
(9823, 982, '删除', '', 2, 'POST /api/service/admin/notice/log/delete', '', 'notice:log:delete', 3, 1, 0, NOW(), NOW()),
|
||
(9824, 982, '清空', '', 2, 'POST /api/service/admin/notice/log/clear', '', 'notice:log:clear', 4, 1, 0, NOW(), NOW()),
|
||
(9825, 982, '选项', '', 2, 'POST /api/service/admin/notice/log/options','', 'notice:log:options',5, 1, 1, NOW(), NOW())
|
||
ON DUPLICATE KEY UPDATE parent_id=VALUES(parent_id), name=VALUES(name), type=VALUES(type),
|
||
path=VALUES(path), permission=VALUES(permission), sort=VALUES(sort), status=VALUES(status),
|
||
hidden=VALUES(hidden), deleted_at=NULL;
|
||
|
||
-- ============ 4) 超管角色绑定 ============
|
||
INSERT IGNORE INTO admin_role_menu (role_id, menu_id, created_at, updated_at)
|
||
SELECT 1, id, NOW(), NOW() FROM admin_menu WHERE id IN (982, 9821, 9822, 9823, 9824, 9825);
|