From 8721d010286b54f196d12336ae46dfb30f0b5158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A4=8F=E7=8A=80=E9=BA=9F?= Date: Thu, 27 Aug 2026 02:26:53 +0800 Subject: [PATCH] =?UTF-8?q?fix(recruitment):=20fetchHTML=20curl=20?= =?UTF-8?q?=E4=BC=98=E5=85=88=20+=20https=20=E5=BD=92=E4=B8=80=E5=8C=96=20?= =?UTF-8?q?+=20=E8=B0=83=E8=AF=95=E6=97=A5=E5=BF=97=E6=94=B9=20warning=20?= =?UTF-8?q?=E7=BA=A7=EF=BC=88logger.level=3Dwarning=20=E8=BF=87=E6=BB=A4?= =?UTF-8?q?=E4=BA=86=20Infof=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/service/recruitment/crawler.go | 39 +++++++++++++------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/internal/service/recruitment/crawler.go b/internal/service/recruitment/crawler.go index 6eb6817..87f103c 100644 --- a/internal/service/recruitment/crawler.go +++ b/internal/service/recruitment/crawler.go @@ -79,10 +79,10 @@ func genericStaticCrawl(ctx context.Context, src entity.CrawlSource, force bool) if err != nil { return nil, err } - g.Log().Infof(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html)) + g.Log().Warningf(ctx, "[recruit-debug] src=%d listURL=%s htmlLen=%d", src.Id, listURL, len(html)) raw := extractAnchors(html, listURL) anchors := filterArticleAnchors(raw) - g.Log().Infof(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors)) + g.Log().Warningf(ctx, "[recruit-debug] src=%d rawAnchors=%d filtered=%d", src.Id, len(raw), len(anchors)) var out []dto.RecruitmentInput limit := 60 if force { @@ -189,10 +189,17 @@ func walkJSON(node any, out *[]dto.RecruitmentInput) { } } -// fetchHTML 带超时与浏览器 UA 的请求，降低被反爬概率。 -// 部分政府站对 Go HTTP 客户端做 TLS 指纹级拦截（返回无锚点页面）， -// 此时回退系统 curl 重试（容器镜像内置 curl，已验证可拿到完整列表页）。 +// fetchHTML 抓取页面。优先用系统 curl（已实测：政府站在容器内对 curl 返回完整页面， +// 而 Go 客户端可能被 TLS 指纹级 WAF 拦截返回无内容挑战页），失败再回退 Go 客户端。 func fetchHTML(ctx context.Context, u string) (string, error) { + if body, e := fetchWithCurl(u); e == nil && looksLikeHtml(body) && len(body) >= 3000 { + g.Log().Warningf(ctx, "[recruit-debug] curl ok len=%d url=%s", len(body), u) + return body, nil + } else if e != nil { + g.Log().Warningf(ctx, "[recruit-debug] curl failed: %v url=%s", e, u) + } else { + g.Log().Warningf(ctx, "[recruit-debug] curl suspicious len=%d url=%s", len(body), u) + } client := g.Client() client.SetTimeout(15 * time.Second) client.SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36") @@ -200,27 +207,18 @@ func fetchHTML(ctx context.Context, u string) (string, error) { client.SetHeader("Accept-Language", "zh-CN,zh;q=0.9") resp, err := client.Get(ctx, u) if err != nil { - // 网络层失败：尝试 curl 兜底，避免单点失败。 - if body, e := fetchWithCurl(u); e == nil { - return body, nil - } return "", gerror.Wrapf(err, "fetch %s", u) } defer resp.Close() if resp.StatusCode != 200 { - // 非 200（如 302/403）：回退 curl 重试（curl 自动跟随重定向）。 - if body, e := fetchWithCurl(u); e == nil { - return body, nil - } return "", gerror.Newf("HTTP %d for %s", resp.StatusCode, u) } body := resp.ReadAllString() - // 内容可疑（疑似 WAF/JS 挑战页：无 html 特征或锚点过少）时回退 curl。 if !looksLikeHtml(body) || len(body) < 3000 { - if cb, e := fetchWithCurl(u); e == nil { - return cb, nil - } + g.Log().Warningf(ctx, "[recruit-debug] go-fallback suspicious len=%d url=%s", len(body), u) + return "", gerror.Newf("suspicious page (len=%d) for %s", len(body), u) } + g.Log().Warningf(ctx, "[recruit-debug] go-fallback ok len=%d url=%s", len(body), u) return body, nil } @@ -421,7 +419,12 @@ func resolveURL(base, href string) string { if err != nil { return "" } - return u.ResolveReference(ref).String() + abs := u.ResolveReference(ref).String() + // 同一政府页面常同时含 http/https 两种文章链接，统一升级为 https 便于去重。 + if strings.HasPrefix(base, "https://") && strings.HasPrefix(abs, "http://") { + abs = "https://" + strings.TrimPrefix(abs, "http://") + } + return abs } // joinURL 拼接基础域名与路径。