638 lines
19 KiB
Go
638 lines
19 KiB
Go
package recruitment
|
||
|
||
import (
|
||
"context"
|
||
"crypto/md5"
|
||
"encoding/json"
|
||
"fmt"
|
||
"net/url"
|
||
"os/exec"
|
||
"regexp"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/gogf/gf/v2/errors/gerror"
|
||
"github.com/gogf/gf/v2/frame/g"
|
||
"github.com/gogf/gf/v2/os/gtime"
|
||
|
||
"service.xpcool.com/internal/dao"
|
||
"service.xpcool.com/internal/model/do"
|
||
"service.xpcool.com/internal/model/dto"
|
||
"service.xpcool.com/internal/model/entity"
|
||
)
|
||
|
||
// crawlerRun 按数据源类型分发抓取，返回运行结果（供写日志/统计）。
|
||
func crawlerRun(ctx context.Context, src entity.CrawlSource, force bool) dto.CrawlRunResult {
|
||
res := dto.CrawlRunResult{SourceId: src.Id}
|
||
defer func() {
|
||
// 防止单源解析panic拖垮整体调度。
|
||
if r := recover(); r != nil {
|
||
res.Err = gerror.Newf("crawler panic: %v", r)
|
||
}
|
||
}()
|
||
var (
|
||
infos []dto.RecruitmentInput
|
||
err error
|
||
)
|
||
switch src.SourceType {
|
||
case 1: // 静态列表页
|
||
infos, err = genericStaticCrawl(ctx, src, force)
|
||
case 2: // SPA / JS 渲染（需底层 JSON 接口，首批 POC）
|
||
infos, err = spaCrawl(ctx, src, force)
|
||
case 3: // 需登录/验证码，首批已排除
|
||
err = gerror.New("该源需登录/验证码，首批已排除")
|
||
case 4: // 附件型，先按静态抓取，附件解析为后续增强
|
||
infos, err = genericStaticCrawl(ctx, src, force)
|
||
default:
|
||
err = gerror.Newf("未知源类型 %d", src.SourceType)
|
||
}
|
||
res.Err = err
|
||
res.Fetched = len(infos)
|
||
if err != nil {
|
||
return res
|
||
}
|
||
newCount, updated := 0, 0
|
||
for i := range infos {
|
||
isNew, e := upsertInfo(ctx, infos[i])
|
||
if e != nil {
|
||
res.Err = e
|
||
continue
|
||
}
|
||
if isNew {
|
||
newCount++
|
||
} else {
|
||
updated++
|
||
}
|
||
}
|
||
res.NewCount = newCount
|
||
res.UpdatedCount = updated
|
||
return res
|
||
}
|
||
|
||
// genericStaticCrawl 通用静态列表抓取：取列表页锚点 → 逐条抓详情 → 抽取字段。
|
||
func genericStaticCrawl(ctx context.Context, src entity.CrawlSource, force bool) ([]dto.RecruitmentInput, error) {
|
||
listURL := joinURL(src.BaseUrl, src.ListPath)
|
||
if listURL == "" {
|
||
listURL = src.BaseUrl
|
||
}
|
||
html, err := fetchHTML(ctx, listURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
g.Log().Infof(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html))
|
||
raw := extractAnchors(html, listURL)
|
||
anchors := filterArticleAnchors(raw)
|
||
g.Log().Infof(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors))
|
||
var out []dto.RecruitmentInput
|
||
limit := 60
|
||
if force {
|
||
limit = 300 // 全量回溯放宽上限
|
||
}
|
||
for _, a := range anchors {
|
||
if len(out) >= limit {
|
||
break
|
||
}
|
||
dHtml, e := fetchHTML(ctx, a.URL)
|
||
if e != nil {
|
||
continue
|
||
}
|
||
title, content, pub, dl, ex := extractDetail(dHtml)
|
||
if title == "" {
|
||
title = a.Text
|
||
}
|
||
pubDate := normalizeDate(pub)
|
||
// 增量模式：跳过 30 天前的公告，避免无效回扫。
|
||
if !force && pubDate != "" {
|
||
if t, e2 := time.Parse("2006-01-02", pubDate); e2 == nil {
|
||
if time.Since(t) > 30*24*time.Hour {
|
||
continue
|
||
}
|
||
}
|
||
}
|
||
out = append(out, dto.RecruitmentInput{
|
||
Title: title,
|
||
SourceId: src.Id,
|
||
OrgName: src.Name,
|
||
Category: src.Category,
|
||
Region: src.Region,
|
||
PublishDate: pubDate,
|
||
Deadline: normalizeDate(dl),
|
||
ExamDate: normalizeDate(ex),
|
||
Url: a.URL,
|
||
Content: content,
|
||
Status: 0,
|
||
})
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// spaCrawl SPA 源抓取（最佳实践：逆向底层 JSON 接口）。当前为占位实现，
|
||
// 若服务端渲染无锚点则返回明确错误，便于在阶段0 POC 中补齐接口。
|
||
func spaCrawl(ctx context.Context, src entity.CrawlSource, force bool) ([]dto.RecruitmentInput, error) {
|
||
listURL := joinURL(src.BaseUrl, src.ListPath)
|
||
if listURL == "" {
|
||
listURL = src.BaseUrl
|
||
}
|
||
html, err := fetchHTML(ctx, listURL)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// 部分 SPA 会在 HTML 内联 __NEXT_DATA__ / 初始 state，可从中抽取。
|
||
if data := extractFromInlineJSON(html); len(data) > 0 {
|
||
return data, nil
|
||
}
|
||
// 纯客户端渲染（hash 路由）无服务端内容，需 POC 接入接口。
|
||
if len(extractAnchors(html, listURL)) == 0 {
|
||
return nil, gerror.New("SPA 无服务端渲染内容，需接入底层 JSON 接口（阶段0 POC）")
|
||
}
|
||
return genericStaticCrawl(ctx, src, force)
|
||
}
|
||
|
||
// extractFromInlineJSON 从 SPA 内联 JSON（__NEXT_DATA__ / window.__INITIAL_STATE__）抽取标题与链接。
|
||
// 适配 Vue/Next 等 SSR/CSR 混合页面，作为 SPA 源的兜底解析。
|
||
func extractFromInlineJSON(html string) []dto.RecruitmentInput {
|
||
var out []dto.RecruitmentInput
|
||
// 匹配常见内联 JSON 块，逐块扫描其中的标题/链接字段。
|
||
blocks := inlineJSONRe.FindAllStringSubmatch(html, -1)
|
||
for _, b := range blocks {
|
||
raw := b[1]
|
||
var generic map[string]any
|
||
if err := json.Unmarshal([]byte(raw), &generic); err != nil {
|
||
continue
|
||
}
|
||
walkJSON(generic, &out)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// walkJSON 递归遍历内联 JSON，提取含标题与链接的条目（最佳实践，避免硬规则）。
|
||
func walkJSON(node any, out *[]dto.RecruitmentInput) {
|
||
switch v := node.(type) {
|
||
case map[string]any:
|
||
title, _ := v["title"].(string)
|
||
link, _ := v["url"].(string)
|
||
if link == "" {
|
||
link, _ = v["href"].(string)
|
||
}
|
||
if title != "" && link != "" {
|
||
*out = append(*out, dto.RecruitmentInput{
|
||
Title: title, Url: link, Status: 0,
|
||
})
|
||
}
|
||
for _, val := range v {
|
||
walkJSON(val, out)
|
||
}
|
||
case []any:
|
||
for _, item := range v {
|
||
walkJSON(item, out)
|
||
}
|
||
}
|
||
}
|
||
|
||
// fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。
|
||
// 部分政府站对 Go HTTP 客户端做 TLS 指纹级拦截（返回无锚点页面），
|
||
// 此时回退系统 curl 重试（容器镜像内置 curl，已验证可拿到完整列表页）。
|
||
func fetchHTML(ctx context.Context, u string) (string, error) {
|
||
client := g.Client()
|
||
client.SetTimeout(15 * time.Second)
|
||
client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
|
||
client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*")
|
||
client.SetHeader("Accept-Language", "zh-CN,zh;q=0.9")
|
||
resp, err := client.Get(ctx, u)
|
||
if err != nil {
|
||
// 网络层失败：尝试 curl 兜底，避免单点失败。
|
||
if body, e := fetchWithCurl(u); e == nil {
|
||
return body, nil
|
||
}
|
||
return "", gerror.Wrapf(err, "fetch %s", u)
|
||
}
|
||
defer resp.Close()
|
||
if resp.StatusCode != 200 {
|
||
// 非 200（如 302/403）：回退 curl 重试（curl 自动跟随重定向）。
|
||
if body, e := fetchWithCurl(u); e == nil {
|
||
return body, nil
|
||
}
|
||
return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u)
|
||
}
|
||
body := resp.ReadAllString()
|
||
// 内容可疑（疑似 WAF/JS 挑战页：无 html 特征或锚点过少）时回退 curl。
|
||
if !looksLikeHtml(body) || len(body) < 3000 {
|
||
if cb, e := fetchWithCurl(u); e == nil {
|
||
return cb, nil
|
||
}
|
||
}
|
||
return body, nil
|
||
}
|
||
|
||
// fetchWithCurl 调用系统 curl 抓取（自动跟随重定向，带浏览器 UA）。
|
||
func fetchWithCurl(u string) (string, error) {
|
||
args := []string{
|
||
"-s", "-L", "--max-time", "25",
|
||
"-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
|
||
"-H", "Accept: text/html,application/xhtml+xml,*/*",
|
||
"-H", "Accept-Language: zh-CN,zh;q=0.9",
|
||
u,
|
||
}
|
||
out, err := exec.Command("curl", args...).Output()
|
||
if err != nil {
|
||
return "", gerror.Wrapf(err, "curl %s", u)
|
||
}
|
||
return string(out), nil
|
||
}
|
||
|
||
// looksLikeHtml 粗略判断响应是否为正常 HTML 文档（含 <a 锚点与 <html 文档标记）。
|
||
func looksLikeHtml(s string) bool {
|
||
lower := strings.ToLower(s)
|
||
return strings.Contains(lower, "<html") && strings.Contains(lower, "<a")
|
||
}
|
||
|
||
type anchor struct {
|
||
URL string
|
||
Text string
|
||
}
|
||
|
||
var (
|
||
// anchorBlockRe 匹配完整 <a>...</a> 块；hrefRe/titleAttrRe 从块内分别抽取链接与 title 属性。
|
||
anchorBlockRe = regexp.MustCompile(`(?is)<a\b[^>]*>([\s\S]*?)</a>`)
|
||
hrefRe = regexp.MustCompile(`(?i)\bhref\s*=\s*["']([^"']+)["']`)
|
||
titleAttrRe = regexp.MustCompile(`(?i)\btitle\s*=\s*["']([^"']+)["']`)
|
||
tagRe = regexp.MustCompile(`<[^>]+>`)
|
||
scriptRe = regexp.MustCompile(`(?is)<script[\s\S]*?</script>`)
|
||
styleRe = regexp.MustCompile(`(?is)<style[\s\S]*?</style>`)
|
||
titleRe = regexp.MustCompile(`(?is)<title[^>]*>([\s\S]*?)</title>`)
|
||
h1Re = regexp.MustCompile(`(?is)<h1[^>]*>([\s\S]*?)</h1>`)
|
||
dateRe = regexp.MustCompile(`(\d{4})[-/年.](\d{1,2})[-/月.](\d{1,2})`)
|
||
inlineJSONRe = regexp.MustCompile(`(?is)(?:__NEXT_DATA__|__INITIAL_STATE__|window\.__[A-Z_]+)\s*=\s*(\{[\s\S]*?\})\s*;?`)
|
||
spaceRe = regexp.MustCompile(`\s+`)
|
||
// recruitKw 招聘类公告常见语义词，用于从政府站链接中识别公告条目。
|
||
recruitKw = regexp.MustCompile(`招聘|招考|公招|选聘|引进|人才|公告|公示|录用|录取|面试|笔试|报名|选调|遴选|拟聘|聘用|招募|招录|考试|考录|体检|考察|资格复审|递补`)
|
||
)
|
||
|
||
// extractAnchors 从 HTML 抽取绝对化后的锚点（URL + 文本）。
|
||
// 政府站常把真实标题放在 title 属性、inner text 仅"详细/更多"，故 inner 过短时回退 title。
|
||
func extractAnchors(html, base string) []anchor {
|
||
out := make([]anchor, 0)
|
||
for _, block := range anchorBlockRe.FindAllStringSubmatch(html, -1) {
|
||
full := block[0]
|
||
hm := hrefRe.FindStringSubmatch(full)
|
||
if len(hm) < 2 {
|
||
continue
|
||
}
|
||
href := strings.TrimSpace(hm[1])
|
||
if href == "" || strings.HasPrefix(href, "javascript:") ||
|
||
strings.HasPrefix(href, "#") || strings.HasPrefix(href, "mailto:") {
|
||
continue
|
||
}
|
||
inner := strings.TrimSpace(stripTags(block[1]))
|
||
text := inner
|
||
if tm := titleAttrRe.FindStringSubmatch(full); len(tm) >= 2 {
|
||
t := strings.TrimSpace(tm[1])
|
||
if len([]rune(text)) < 4 && t != "" {
|
||
text = t
|
||
}
|
||
}
|
||
if text == "" {
|
||
continue
|
||
}
|
||
abs := resolveURL(base, href)
|
||
if abs == "" {
|
||
continue
|
||
}
|
||
out = append(out, anchor{URL: abs, Text: text})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// filterArticleAnchors 过滤导航/无用链接，保留疑似招聘公告条目并去重。
|
||
func filterArticleAnchors(as []anchor) []anchor {
|
||
out := make([]anchor, 0, len(as))
|
||
for _, a := range as {
|
||
text := strings.TrimSpace(a.Text)
|
||
if len([]rune(text)) < 4 {
|
||
continue
|
||
}
|
||
u := strings.ToLower(a.URL)
|
||
// URL 形态线索：静态详情页、政务公开/人事招考栏目、含年份。
|
||
urlHint := strings.Contains(u, ".html") || strings.Contains(u, ".shtml") ||
|
||
strings.Contains(u, ".php") || strings.Contains(u, "/tzgg") ||
|
||
strings.Contains(u, "/rszk") || strings.Contains(u, "/rsxx") ||
|
||
strings.Contains(u, "/zfxx") || dateRe.MatchString(a.URL)
|
||
// 文本语义线索：政府站公告标题常含这些词（必须命中，过滤"市长信箱/重点领域"等非招聘页）。
|
||
textHint := recruitKw.MatchString(text) || recruitKw.MatchString(a.URL)
|
||
// 同时满足"内容页形态"与"招聘语义"，避免误抓栏目/互动页。
|
||
if urlHint && textHint {
|
||
out = append(out, a)
|
||
}
|
||
}
|
||
seen := map[string]bool{}
|
||
uniq := out[:0]
|
||
for _, a := range out {
|
||
if seen[a.URL] {
|
||
continue
|
||
}
|
||
seen[a.URL] = true
|
||
uniq = append(uniq, a)
|
||
}
|
||
return uniq
|
||
}
|
||
|
||
// extractDetail 从详情页抽取标题/正文/发布日期/报名截止/笔试日期（启发式，可按源调优）。
|
||
func extractDetail(html string) (title, content, publishDate, deadline, examDate string) {
|
||
if m := titleRe.FindStringSubmatch(html); len(m) > 1 {
|
||
title = stripTags(m[1])
|
||
}
|
||
if title == "" {
|
||
if m := h1Re.FindStringSubmatch(html); len(m) > 1 {
|
||
title = stripTags(m[1])
|
||
}
|
||
}
|
||
title = strings.TrimSpace(title)
|
||
body := scriptRe.ReplaceAllString(html, " ")
|
||
body = styleRe.ReplaceAllString(body, " ")
|
||
text := stripTags(body)
|
||
text = collapseSpace(text)
|
||
if len([]rune(text)) > 4000 {
|
||
text = string([]rune(text)[:4000])
|
||
}
|
||
content = text
|
||
publishDate = normalizeDate(firstDate(html))
|
||
deadline = normalizeDate(findDateAfter(text, "报名"))
|
||
examDate = normalizeDate(findDateAfter(text, "笔试"))
|
||
return
|
||
}
|
||
|
||
// firstDate 返回文本中首个日期（归一化后）。
|
||
func firstDate(s string) string {
|
||
m := dateRe.FindStringSubmatch(s)
|
||
if len(m) == 0 {
|
||
return ""
|
||
}
|
||
return normalizeDate(m[0])
|
||
}
|
||
|
||
// findDateAfter 在 keyword 之后查找下一个日期（用于报名/笔试时间）。
|
||
func findDateAfter(text, keyword string) string {
|
||
idx := strings.Index(text, keyword)
|
||
if idx < 0 {
|
||
return ""
|
||
}
|
||
rest := text[idx:]
|
||
m := dateRe.FindStringSubmatch(rest)
|
||
if len(m) == 0 {
|
||
return ""
|
||
}
|
||
return normalizeDate(m[0])
|
||
}
|
||
|
||
// normalizeDate 将多种中文/西式日期归一化为 YYYY-MM-DD。
|
||
func normalizeDate(s string) string {
|
||
s = strings.TrimSpace(s)
|
||
if s == "" {
|
||
return ""
|
||
}
|
||
m := dateRe.FindStringSubmatch(s)
|
||
if len(m) == 0 {
|
||
return ""
|
||
}
|
||
y, mo, d := m[1], atoiSafe(m[2]), atoiSafe(m[3])
|
||
return fmt.Sprintf("%s-%02d-%02d", y, mo, d)
|
||
}
|
||
|
||
// stripTags 去除 HTML 标签。
|
||
func stripTags(s string) string {
|
||
return tagRe.ReplaceAllString(s, " ")
|
||
}
|
||
|
||
// collapseSpace 折叠空白字符。
|
||
func collapseSpace(s string) string {
|
||
return spaceRe.ReplaceAllString(s, " ")
|
||
}
|
||
|
||
// resolveURL 将相对链接解析为绝对 URL。
|
||
func resolveURL(base, href string) string {
|
||
if strings.HasPrefix(href, "http://") || strings.HasPrefix(href, "https://") {
|
||
return href
|
||
}
|
||
u, err := url.Parse(base)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
ref, err := url.Parse(href)
|
||
if err != nil {
|
||
return ""
|
||
}
|
||
return u.ResolveReference(ref).String()
|
||
}
|
||
|
||
// joinURL 拼接基础域名与路径。
|
||
func joinURL(base, p string) string {
|
||
if p == "" {
|
||
return base
|
||
}
|
||
if strings.HasPrefix(p, "http") {
|
||
return p
|
||
}
|
||
u, err := url.Parse(base)
|
||
if err != nil {
|
||
return base + p
|
||
}
|
||
ref, err := url.Parse(p)
|
||
if err != nil {
|
||
return base + p
|
||
}
|
||
return u.ResolveReference(ref).String()
|
||
}
|
||
|
||
func atoiSafe(s string) int {
|
||
n := 0
|
||
for _, c := range s {
|
||
if c < '0' || c > '9' {
|
||
break
|
||
}
|
||
n = n*10 + int(c-'0')
|
||
}
|
||
return n
|
||
}
|
||
|
||
// ---------- 去重与写入 ----------
|
||
|
||
func fingerprintOf(in dto.RecruitmentInput) string {
|
||
h := md5.Sum([]byte(fmt.Sprintf("%d|%s|%s|%s", in.SourceId, in.Title, in.PublishDate, in.Url)))
|
||
return fmt.Sprintf("%x", h)
|
||
}
|
||
|
||
func groupKeyOf(in dto.RecruitmentInput) string {
|
||
h := md5.Sum([]byte(fmt.Sprintf("%s|%s", in.Title, in.PublishDate)))
|
||
return fmt.Sprintf("%x", h)
|
||
}
|
||
|
||
// upsertInfo 按指纹去重写入公告；已存在则更新可变字段（内容/状态/日期）。
|
||
func upsertInfo(ctx context.Context, in dto.RecruitmentInput) (bool, error) {
|
||
if in.Fingerprint == "" {
|
||
in.Fingerprint = fingerprintOf(in)
|
||
}
|
||
if in.GroupKey == "" {
|
||
in.GroupKey = groupKeyOf(in)
|
||
}
|
||
var exist entity.RecruitmentInfo
|
||
if err := dao.RecruitmentInfo.Ctx(ctx).Where("fingerprint", in.Fingerprint).Scan(&exist); err != nil {
|
||
// gf 的 Scan 在查无记录时返回 "no rows" 错误，视为未存在，继续插入。
|
||
if !strings.Contains(err.Error(), "no rows") {
|
||
return false, gerror.Wrap(err, "query exist info")
|
||
}
|
||
}
|
||
if exist.Id > 0 {
|
||
_, err := dao.RecruitmentInfo.Ctx(ctx).Where("id", exist.Id).Data(do.RecruitmentInfo{
|
||
Content: in.Content, Status: in.Status, Deadline: in.Deadline,
|
||
ExamDate: in.ExamDate, UpdatedAt: gtime.Now(),
|
||
}).Update()
|
||
if err != nil {
|
||
return false, gerror.Wrap(err, "update info")
|
||
}
|
||
return false, nil
|
||
}
|
||
orgId, err := upsertOrg(ctx, in.OrgName, in.Category, in.Region)
|
||
if err != nil {
|
||
return false, err
|
||
}
|
||
in.OrgId = orgId
|
||
if _, err = dao.RecruitmentInfo.Ctx(ctx).Data(toRecruitmentDO(in)).InsertAndGetId(); err != nil {
|
||
return false, gerror.Wrap(err, "insert info")
|
||
}
|
||
return true, nil
|
||
}
|
||
|
||
// upsertOrg 发布主体按名称去重，返回主键。
|
||
func upsertOrg(ctx context.Context, name string, category int, region string) (uint64, error) {
|
||
if name == "" {
|
||
return 0, nil
|
||
}
|
||
var org entity.Organization
|
||
if err := dao.Organization.Ctx(ctx).Where("name", name).Scan(&org); err != nil {
|
||
if !strings.Contains(err.Error(), "no rows") {
|
||
return 0, gerror.Wrap(err, "query org")
|
||
}
|
||
}
|
||
if org.Id > 0 {
|
||
return org.Id, nil
|
||
}
|
||
id, err := dao.Organization.Ctx(ctx).Data(do.Organization{
|
||
Name: name, Type: mapCategoryToOrgType(category), Region: region,
|
||
CreatedAt: gtime.Now(), UpdatedAt: gtime.Now(),
|
||
}).InsertAndGetId()
|
||
if err != nil {
|
||
return 0, gerror.Wrap(err, "insert org")
|
||
}
|
||
return uint64(id), nil
|
||
}
|
||
|
||
func mapCategoryToOrgType(c int) int {
|
||
switch c {
|
||
case 2:
|
||
return 2 // 事业单位
|
||
case 3:
|
||
return 3 // 国企
|
||
case 4:
|
||
return 4 // 央企
|
||
case 5:
|
||
return 5 // 私企
|
||
default:
|
||
return 6
|
||
}
|
||
}
|
||
|
||
func toRecruitmentDO(in dto.RecruitmentInput) do.RecruitmentInfo {
|
||
now := gtime.Now()
|
||
return do.RecruitmentInfo{
|
||
Title: in.Title, SourceId: in.SourceId, OrgId: in.OrgId, OrgName: in.OrgName,
|
||
Category: in.Category, Region: in.Region, PublishDate: nullIfEmpty(in.PublishDate),
|
||
Deadline: nullIfEmpty(in.Deadline), ExamDate: nullIfEmpty(in.ExamDate), Url: in.Url, Content: in.Content,
|
||
Attachments: in.Attachments, Status: in.Status, Fingerprint: in.Fingerprint,
|
||
GroupKey: in.GroupKey, CreatedAt: now, UpdatedAt: now,
|
||
}
|
||
}
|
||
|
||
// nullIfEmpty 将空字符串转为 nil，便于插入 NULL（DATE 等可空列不接受空串）。
|
||
func nullIfEmpty(s string) any {
|
||
if strings.TrimSpace(s) == "" {
|
||
return nil
|
||
}
|
||
return s
|
||
}
|
||
|
||
// recordCrawlLog 写抓取日志并更新数据源运行状态（成功清失败计数；失败累加）。
|
||
func recordCrawlLog(ctx context.Context, res dto.CrawlRunResult) {
|
||
errMsg := ""
|
||
if res.Err != nil {
|
||
// 错误可能含整页 SQL，截断避免超过 error 列长度导致日志也写不进。
|
||
errMsg = res.Err.Error()
|
||
const maxErr = 900
|
||
if len(errMsg) > maxErr {
|
||
errMsg = errMsg[:maxErr] + "...(truncated)"
|
||
}
|
||
}
|
||
if _, err := dao.CrawlLog.Ctx(ctx).Data(do.CrawlLog{
|
||
SourceId: res.SourceId, RunAt: gtime.Now(), Fetched: res.Fetched,
|
||
NewCount: res.NewCount, UpdatedCount: res.UpdatedCount, Error: errMsg,
|
||
}).Insert(); err != nil {
|
||
g.Log().Errorf(ctx, "write crawl log failed: %v", err)
|
||
}
|
||
if res.Err != nil {
|
||
var src entity.CrawlSource
|
||
_ = dao.CrawlSource.Ctx(ctx).Where("id", res.SourceId).Scan(&src)
|
||
_, _ = dao.CrawlSource.Ctx(ctx).Where("id", res.SourceId).
|
||
Data(do.CrawlSource{FailCount: src.FailCount + 1}).Update()
|
||
return
|
||
}
|
||
_, _ = dao.CrawlSource.Ctx(ctx).Where("id", res.SourceId).
|
||
Data(do.CrawlSource{LastSuccessAt: gtime.Now().String(), FailCount: 0}).Update()
|
||
}
|
||
|
||
// ---------- 订阅辅助 ----------
|
||
|
||
// loadSubscription 加载推送订阅：id>0 按 id；否则取首个启用订阅。
|
||
func loadSubscription(ctx context.Context, id uint64) (*entity.PushSubscription, error) {
|
||
var sub entity.PushSubscription
|
||
m := dao.PushSubscription.Ctx(ctx)
|
||
if id > 0 {
|
||
m = m.Where("id", id)
|
||
} else {
|
||
m = m.Where("enabled", 1)
|
||
}
|
||
if err := m.OrderDesc("id").Scan(&sub); err != nil {
|
||
return nil, gerror.Wrap(err, "query subscription")
|
||
}
|
||
if sub.Id == 0 {
|
||
return nil, nil
|
||
}
|
||
return &sub, nil
|
||
}
|
||
|
||
func doPushSubscription(in dto.SubscriptionInput) do.PushSubscription {
|
||
regions, _ := json.Marshal(in.Regions)
|
||
cats, _ := json.Marshal(in.Categories)
|
||
return do.PushSubscription{
|
||
Name: in.Name, DeviceKey: in.DeviceKey, Regions: string(regions),
|
||
Categories: string(cats), OnlyNew: in.OnlyNew, PushTime: in.PushTime,
|
||
Enabled: in.Enabled, UpdatedAt: gtime.Now(),
|
||
}
|
||
}
|
||
|
||
func parseJSONStrings(s string) []string {
|
||
if s == "" {
|
||
return nil
|
||
}
|
||
var out []string
|
||
_ = json.Unmarshal([]byte(s), &out)
|
||
return out
|
||
}
|
||
|
||
func parseJSONInts(s string) []int {
|
||
if s == "" {
|
||
return nil
|
||
}
|
||
var out []int
|
||
_ = json.Unmarshal([]byte(s), &out)
|
||
return out
|
||
}
|