xpcool/service.xpcool.com
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: xpcool:1e28e02dc1305d3f3c537c207f3df1151ba7b724
Branches Tags
xpcool:dev
xpcool:main
...
compare: xpcool:8721d010286b54f196d12336ae46dfb30f0b5158
Branches Tags
xpcool:main
xpcool:dev

2 Commits
1e28e02dc1 ... 8721d01028

Author SHA1 Message Date
夏犀麟
8721d01028 fix(recruitment): fetchHTML curl 优先 + https 归一化 + 调试日志改 warning 级（logger.level=warning 过滤了 Infof）
Some checks failed
Build and Deploy (service.xpcool.com) / build-and-deploy (push) Failing after 32s
Details
2026-08-27 02:26:53 +08:00
夏犀麟
8b11893d35 feat(recruitment): 爬虫修复与部署打通 - Scan no-rows 入库修复 / title 属性兜底 / 双条件过滤 / 空日期插 NULL / crawl_log 截断 / fetchHTML curl 兜底 / 人社局改 https / 新增 config.yaml 2026-08-27 02:12:03 +08:00
1 changed files with 49 additions and 5 deletions
Show Stats Expand all files Collapse all files

54
internal/service/recruitment/crawler.go
View File

@ -6,6 +6,7 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"net/url" "net/url"
"os/exec"
"regexp" "regexp"
"strings" "strings"
"time" "time"
@ -78,10 +79,10 @@ func genericStaticCrawl(ctx context.Context, src entity.CrawlSource, force bool)
if err != nil { if err != nil {
return nil, err return nil, err
} }
g.Log().Infof(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html)) g.Log().Warningf(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html))
raw := extractAnchors(html, listURL) raw := extractAnchors(html, listURL)
anchors := filterArticleAnchors(raw) anchors := filterArticleAnchors(raw)
g.Log().Infof(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors)) g.Log().Warningf(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors))
var out []dto.RecruitmentInput var out []dto.RecruitmentInput
limit := 60 limit := 60
if force { if force {
@ -188,12 +189,22 @@ func walkJSON(node any, out *[]dto.RecruitmentInput) {
} }
} }
// fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。 // fetchHTML 抓取页面。优先用系统 curl（已实测：政府站在容器内对 curl 返回完整页面，
// 而 Go 客户端可能被 TLS 指纹级 WAF 拦截返回无内容挑战页），失败再回退 Go 客户端。
func fetchHTML(ctx context.Context, u string) (string, error) { func fetchHTML(ctx context.Context, u string) (string, error) {
if body, e := fetchWithCurl(u); e == nil && looksLikeHtml(body) && len(body) >= 3000 {
g.Log().Warningf(ctx, "[recruit-debug] curl ok len=%d url=%s", len(body), u)
return body, nil
} else if e != nil {
g.Log().Warningf(ctx, "[recruit-debug] curl failed: %v url=%s", e, u)
} else {
g.Log().Warningf(ctx, "[recruit-debug] curl suspicious len=%d url=%s", len(body), u)
}
client := g.Client() client := g.Client()
client.SetTimeout(15 * time.Second) client.SetTimeout(15 * time.Second)
client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36") client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*") client.SetHeader("Accept", "text/html,application/xhtml+xml,*/*")
client.SetHeader("Accept-Language", "zh-CN,zh;q=0.9")
resp, err := client.Get(ctx, u) resp, err := client.Get(ctx, u)
if err != nil { if err != nil {
return "", gerror.Wrapf(err, "fetch %s", u) return "", gerror.Wrapf(err, "fetch %s", u)
@ -202,7 +213,35 @@ func fetchHTML(ctx context.Context, u string) (string, error) {
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u) return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u)
} }
return resp.ReadAllString(), nil body := resp.ReadAllString()
if !looksLikeHtml(body) || len(body) < 3000 {
g.Log().Warningf(ctx, "[recruit-debug] go-fallback suspicious len=%d url=%s", len(body), u)
return "", gerror.Newf("suspicious page (len=%d) for %s", len(body), u)
}
g.Log().Warningf(ctx, "[recruit-debug] go-fallback ok len=%d url=%s", len(body), u)
return body, nil
}
// fetchWithCurl 调用系统 curl 抓取（自动跟随重定向，带浏览器 UA）。
func fetchWithCurl(u string) (string, error) {
args := []string{
"-s", "-L", "--max-time", "25",
"-A", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
"-H", "Accept: text/html,application/xhtml+xml,*/*",
"-H", "Accept-Language: zh-CN,zh;q=0.9",
u,
}
out, err := exec.Command("curl", args...).Output()
if err != nil {
return "", gerror.Wrapf(err, "curl %s", u)
}
return string(out), nil
}
// looksLikeHtml 粗略判断响应是否为正常 HTML 文档（含 <a 锚点与 <html 文档标记）。
func looksLikeHtml(s string) bool {
lower := strings.ToLower(s)
return strings.Contains(lower, "<html") && strings.Contains(lower, "<a")
} }
type anchor struct { type anchor struct {
@ -380,7 +419,12 @@ func resolveURL(base, href string) string {
if err != nil { if err != nil {
return "" return ""
} }
return u.ResolveReference(ref).String() abs := u.ResolveReference(ref).String()
// 同一政府页面常同时含 http/https 两种文章链接，统一升级为 https 便于去重。
if strings.HasPrefix(base, "https://") && strings.HasPrefix(abs, "http://") {
abs = "https://" + strings.TrimPrefix(abs, "http://")
}
return abs
} }
// joinURL 拼接基础域名与路径。 // joinURL 拼接基础域名与路径。